2024-08-31T08:13:46.591 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-08-31T08:13:46.613 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-31T08:13:46.719 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi/7883237 branch: wip-dis-testing description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff} email: idryomov@gmail.com first_in_suite: false job_id: '7883237' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-dis-testing ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: none client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-dis-testing sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 owner: scheduled_dis@teuthology priority: 75 repo: https://github.com/ceph/ceph-ci roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 4871 sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 sleep_before_teardown: 0 subset: 98/100 suite: rbd suite_branch: wip-dis-testing suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci suite_sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 targets: smithi012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNEx9XsXup32GoJWsUuPxH1n47mKHEI6yaBmm41mtD7h8aNmNH7iD1ARqK8OBx22xCY1NFPDkU2kwMBJEqjSn7E= smithi167.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpbZp5xuE44w4oUgvkpwPbPMuNfddfjlOsUe5biVwgLuxRVcedQHT4m1bf7uePACsma12cDwmAwkJhAgE8y+p8= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-08-31_07:23:49 tube: smithi user: dis verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-31T08:13:46.719 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa; will attempt to use it 2024-08-31T08:13:46.720 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks 2024-08-31T08:13:46.720 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-31T08:13:46.722 INFO:teuthology.task.internal:Checking packages... 2024-08-31T08:13:46.744 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'dc9f1ccbfcab74acda17f39d6ab018266071e760' 2024-08-31T08:13:46.744 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-31T08:13:46.744 INFO:teuthology.packaging:ref: None 2024-08-31T08:13:46.744 INFO:teuthology.packaging:tag: None 2024-08-31T08:13:46.744 INFO:teuthology.packaging:branch: wip-dis-testing 2024-08-31T08:13:46.744 INFO:teuthology.packaging:sha1: dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:13:46.744 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-dis-testing 2024-08-31T08:13:46.925 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4620.gdc9f1ccb 2024-08-31T08:13:46.926 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-31T08:13:46.934 INFO:teuthology.task.internal:no buildpackages task found 2024-08-31T08:13:46.934 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-31T08:13:47.046 INFO:teuthology.task.internal:Saving configuration 2024-08-31T08:13:47.063 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-31T08:13:47.093 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-31T08:13:47.126 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi/7883237', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-31 08:08:44.381682', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNEx9XsXup32GoJWsUuPxH1n47mKHEI6yaBmm41mtD7h8aNmNH7iD1ARqK8OBx22xCY1NFPDkU2kwMBJEqjSn7E='} 2024-08-31T08:13:47.157 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi167.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi/7883237', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-31 08:08:44.383632', 'locked_by': 'scheduled_dis@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpbZp5xuE44w4oUgvkpwPbPMuNfddfjlOsUe5biVwgLuxRVcedQHT4m1bf7uePACsma12cDwmAwkJhAgE8y+p8='} 2024-08-31T08:13:47.157 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-31T08:13:47.164 INFO:teuthology.task.internal:roles: ubuntu@smithi012.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2024-08-31T08:13:47.164 INFO:teuthology.task.internal:roles: ubuntu@smithi167.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2024-08-31T08:13:47.165 INFO:teuthology.run_tasks:Running task console_log... 2024-08-31T08:13:47.241 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7facae883ac0>, signals=[15]) 2024-08-31T08:13:47.242 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-31T08:13:47.249 INFO:teuthology.task.internal:Opening connections... 2024-08-31T08:13:47.249 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:13:47.250 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T08:13:47.337 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:13:47.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T08:13:47.420 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-31T08:13:47.427 DEBUG:teuthology.orchestra.run.smithi012:> uname -m 2024-08-31T08:13:47.444 INFO:teuthology.orchestra.run.smithi012.stdout:x86_64 2024-08-31T08:13:47.444 DEBUG:teuthology.orchestra.run.smithi012:> cat /etc/os-release 2024-08-31T08:13:47.501 INFO:teuthology.orchestra.run.smithi012.stdout:NAME="CentOS Stream" 2024-08-31T08:13:47.501 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION="9" 2024-08-31T08:13:47.501 INFO:teuthology.orchestra.run.smithi012.stdout:ID="centos" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:ID_LIKE="rhel fedora" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION_ID="9" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:PLATFORM_ID="platform:el9" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:ANSI_COLOR="0;31" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:LOGO="fedora-logo-icon" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:HOME_URL="https://centos.org/" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-31T08:13:47.502 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-31T08:13:47.503 INFO:teuthology.lock.ops:Updating smithi012.front.sepia.ceph.com on lock server 2024-08-31T08:13:47.528 DEBUG:teuthology.orchestra.run.smithi167:> uname -m 2024-08-31T08:13:47.546 INFO:teuthology.orchestra.run.smithi167.stdout:x86_64 2024-08-31T08:13:47.547 DEBUG:teuthology.orchestra.run.smithi167:> cat /etc/os-release 2024-08-31T08:13:47.604 INFO:teuthology.orchestra.run.smithi167.stdout:NAME="CentOS Stream" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION="9" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:ID="centos" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:ID_LIKE="rhel fedora" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:VERSION_ID="9" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:PLATFORM_ID="platform:el9" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:ANSI_COLOR="0;31" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:LOGO="fedora-logo-icon" 2024-08-31T08:13:47.605 INFO:teuthology.orchestra.run.smithi167.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-31T08:13:47.606 INFO:teuthology.orchestra.run.smithi167.stdout:HOME_URL="https://centos.org/" 2024-08-31T08:13:47.606 INFO:teuthology.orchestra.run.smithi167.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-08-31T08:13:47.606 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-31T08:13:47.606 INFO:teuthology.orchestra.run.smithi167.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-31T08:13:47.606 INFO:teuthology.lock.ops:Updating smithi167.front.sepia.ceph.com on lock server 2024-08-31T08:13:47.637 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-31T08:13:47.647 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-31T08:13:47.659 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-31T08:13:47.659 DEBUG:teuthology.orchestra.run.smithi012:> test '!' -e /home/ubuntu/cephtest 2024-08-31T08:13:47.661 DEBUG:teuthology.orchestra.run.smithi167:> test '!' -e /home/ubuntu/cephtest 2024-08-31T08:13:47.680 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-31T08:13:47.688 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-31T08:13:47.688 DEBUG:teuthology.orchestra.run.smithi012:> test -z $(ls -A /var/lib/ceph) 2024-08-31T08:13:47.718 DEBUG:teuthology.orchestra.run.smithi167:> test -z $(ls -A /var/lib/ceph) 2024-08-31T08:13:47.817 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-31T08:13:47.885 INFO:teuthology.run_tasks:Running task kernel... 2024-08-31T08:13:47.901 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-31T08:13:47.901 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'kdb': 1, 'sha1': 'distro'}, 'cluster2.mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-31T08:13:47.901 DEBUG:teuthology.orchestra.run.smithi012:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-31T08:13:47.902 DEBUG:teuthology.orchestra.run.smithi167:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-31T08:13:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T08:13:47.919 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-08-31T08:13:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T08:13:47.920 DEBUG:teuthology.orchestra.run.smithi167:> uname -r 2024-08-31T08:13:47.977 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-503.el9.x86_64 2024-08-31T08:13:47.977 INFO:teuthology.task.kernel:Running kernel on smithi012: 5.14.0-503.el9.x86_64 2024-08-31T08:13:47.978 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum install -y kernel 2024-08-31T08:13:47.979 INFO:teuthology.orchestra.run.smithi167.stdout:5.14.0-503.el9.x86_64 2024-08-31T08:13:47.979 INFO:teuthology.task.kernel:Running kernel on smithi167: 5.14.0-503.el9.x86_64 2024-08-31T08:13:47.979 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum install -y kernel 2024-08-31T08:13:50.957 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - BaseOS 6.7 MB/s | 8.2 MB 00:01 2024-08-31T08:13:54.167 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - AppStream 9.9 MB/s | 20 MB 00:02 2024-08-31T08:13:54.185 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 1.8 MB/s | 8.2 MB 00:04 2024-08-31T08:13:56.336 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 21 MB/s | 20 MB 00:00 2024-08-31T08:13:59.076 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - CRB 6.7 MB/s | 6.6 MB 00:00 2024-08-31T08:14:00.668 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 18 kB 00:00 2024-08-31T08:14:01.456 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 34 MB/s | 22 MB 00:00 2024-08-31T08:14:01.878 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 4.0 MB/s | 6.6 MB 00:01 2024-08-31T08:14:03.862 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 27 kB/s | 18 kB 00:00 2024-08-31T08:14:04.791 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 28 MB/s | 22 MB 00:00 2024-08-31T08:14:07.053 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 23 kB/s | 1.7 kB 00:00 2024-08-31T08:14:09.110 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-31T08:14:09.110 INFO:teuthology.orchestra.run.smithi167.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-31T08:14:09.188 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:14:09.189 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:14:09.189 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:14:09.293 DEBUG:teuthology.orchestra.run.smithi167:> echo no | sudo yum reinstall kernel || true 2024-08-31T08:14:09.836 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:02 ago on Sat 31 Aug 2024 08:14:07 AM UTC. 2024-08-31T08:14:09.976 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:14:09.984 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalling: 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:09.985 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:09.986 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 1.9 M 2024-08-31T08:14:09.986 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 0 2024-08-31T08:14:09.986 INFO:teuthology.orchestra.run.smithi167.stderr:Operation aborted. 2024-08-31T08:14:10.050 INFO:teuthology.orchestra.run.smithi167.stdout:Is this ok [y/N]: 2024-08-31T08:14:10.050 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum reinstall -y kernel || true 2024-08-31T08:14:10.413 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-08-31T08:14:10.584 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:03 ago on Sat 31 Aug 2024 08:14:07 AM UTC. 2024-08-31T08:14:10.710 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:14:10.710 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalling: 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 1.9 M 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 0 2024-08-31T08:14:10.711 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:14:11.089 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-5.14.0-503.el9.x86_64.rpm 5.7 MB/s | 1.9 MB 00:00 2024-08-31T08:14:11.089 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:14:11.089 INFO:teuthology.orchestra.run.smithi167.stdout:Total 5.0 MB/s | 1.9 MB 00:00 2024-08-31T08:14:11.117 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:14:11.351 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:14:11.351 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:14:11.381 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:14:11.381 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:14:11.555 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:14:11.650 INFO:teuthology.orchestra.run.smithi167.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T08:14:11.650 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:12.422 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:12.422 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T08:14:12.503 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-31T08:14:12.503 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-08-31T08:14:12.592 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:14:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:14:12.593 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:14:12.718 DEBUG:teuthology.orchestra.run.smithi012:> echo no | sudo yum reinstall kernel || true 2024-08-31T08:14:13.111 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:13.111 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:13.111 INFO:teuthology.orchestra.run.smithi167.stdout:Reinstalled: 2024-08-31T08:14:13.111 INFO:teuthology.orchestra.run.smithi167.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-31T08:14:13.112 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:14:13.112 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:14:13.269 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:03 ago on Sat 31 Aug 2024 08:14:10 AM UTC. 2024-08-31T08:14:13.310 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q kernel | sort -rV | head -n 1 2024-08-31T08:14:13.369 INFO:teuthology.orchestra.run.smithi167.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-31T08:14:13.369 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-31T08:14:13.369 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-31T08:14:13.369 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2024-08-31T08:14:13.369 DEBUG:teuthology.orchestra.run.smithi167:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-31T08:14:13.404 INFO:teuthology.orchestra.run.smithi167.stdout:ttyS1 2024-08-31T08:14:13.431 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:14:13.432 DEBUG:teuthology.parallel:result is None 2024-08-31T08:14:13.443 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:13.443 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-08-31T08:14:13.443 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:13.443 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalling: 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 1.9 M 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 0 2024-08-31T08:14:13.444 INFO:teuthology.orchestra.run.smithi012.stderr:Operation aborted. 2024-08-31T08:14:13.506 INFO:teuthology.orchestra.run.smithi012.stdout:Is this ok [y/N]: 2024-08-31T08:14:13.507 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum reinstall -y kernel || true 2024-08-31T08:14:14.058 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:04 ago on Sat 31 Aug 2024 08:14:10 AM UTC. 2024-08-31T08:14:14.189 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:14:14.190 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:14.190 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-08-31T08:14:14.190 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalling: 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-503.el9 baseos 1.9 M 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 1.9 M 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 0 2024-08-31T08:14:14.191 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:14:15.348 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-503.el9.x86_64.rpm 2.2 MB/s | 1.9 MB 00:00 2024-08-31T08:14:15.348 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:14:15.348 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.6 MB/s | 1.9 MB 00:01 2024-08-31T08:14:15.388 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:14:15.635 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:14:15.635 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:14:15.667 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:14:15.668 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:14:15.846 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:14:15.925 INFO:teuthology.orchestra.run.smithi012.stdout: Reinstalling : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T08:14:15.925 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:16.637 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:16.637 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 1/2 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-503.el9.x86_64 2/2 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout:Reinstalled: 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-5.14.0-503.el9.x86_64 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:14:17.327 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:14:17.536 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-08-31T08:14:17.595 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-503.el9.x86_64 2024-08-31T08:14:17.596 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-503.el9.x86_64 2024-08-31T08:14:17.596 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-08-31T08:14:17.596 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2024-08-31T08:14:17.596 DEBUG:teuthology.orchestra.run.smithi012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-31T08:14:17.640 INFO:teuthology.orchestra.run.smithi012.stdout:ttyS1 2024-08-31T08:14:17.662 DEBUG:teuthology.parallel:result is None 2024-08-31T08:14:17.663 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-31T08:14:17.669 INFO:teuthology.task.internal:Creating test directory... 2024-08-31T08:14:17.669 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-31T08:14:17.704 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-31T08:14:17.723 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-31T08:14:17.729 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-31T08:14:17.782 INFO:teuthology.task.internal:Creating archive directory... 2024-08-31T08:14:17.783 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-31T08:14:17.785 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-31T08:14:17.819 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-31T08:14:17.825 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-31T08:14:17.825 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-31T08:14:17.850 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-31T08:14:17.882 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T08:14:17.892 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T08:14:17.909 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T08:14:17.921 INFO:teuthology.orchestra.run.smithi167.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-31T08:14:17.923 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-31T08:14:17.929 INFO:teuthology.task.internal:Configuring sudo... 2024-08-31T08:14:17.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-31T08:14:17.936 DEBUG:teuthology.orchestra.run.smithi167:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-31T08:14:17.963 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-31T08:14:17.972 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-31T08:14:17.972 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-31T08:14:18.002 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-31T08:14:18.019 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T08:14:18.104 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T08:14:18.148 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T08:14:18.233 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T08:14:18.270 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:14:18.270 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-31T08:14:18.334 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T08:14:18.389 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-31T08:14:18.429 DEBUG:teuthology.orchestra.run.smithi167:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T08:14:18.519 DEBUG:teuthology.orchestra.run.smithi167:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-31T08:14:18.553 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:14:18.553 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-31T08:14:18.619 DEBUG:teuthology.orchestra.run.smithi012:> sudo service rsyslog restart 2024-08-31T08:14:18.622 DEBUG:teuthology.orchestra.run.smithi167:> sudo service rsyslog restart 2024-08-31T08:14:18.657 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T08:14:18.701 INFO:teuthology.orchestra.run.smithi167.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-31T08:14:19.080 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-31T08:14:19.088 INFO:teuthology.task.internal:Starting timer... 2024-08-31T08:14:19.089 INFO:teuthology.run_tasks:Running task pcp... 2024-08-31T08:14:19.110 INFO:teuthology.run_tasks:Running task selinux... 2024-08-31T08:14:19.154 DEBUG:teuthology.orchestra.run.smithi012:> sudo service auditd rotate 2024-08-31T08:14:19.190 INFO:teuthology.orchestra.run.smithi012.stdout:Rotating logs: 2024-08-31T08:14:19.192 DEBUG:teuthology.orchestra.run.smithi167:> sudo service auditd rotate 2024-08-31T08:14:19.254 INFO:teuthology.orchestra.run.smithi167.stdout:Rotating logs: 2024-08-31T08:14:19.256 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-31T08:14:19.256 DEBUG:teuthology.orchestra.run.smithi012:> /usr/sbin/getenforce 2024-08-31T08:14:19.274 INFO:teuthology.orchestra.run.smithi012.stdout:Permissive 2024-08-31T08:14:19.275 DEBUG:teuthology.orchestra.run.smithi167:> /usr/sbin/getenforce 2024-08-31T08:14:19.298 INFO:teuthology.orchestra.run.smithi167.stdout:Permissive 2024-08-31T08:14:19.298 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi012.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi167.front.sepia.ceph.com': 'permissive'} 2024-08-31T08:14:19.298 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T08:14:19.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T08:14:19.341 DEBUG:teuthology.orchestra.run.smithi167:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2024-08-31T08:14:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T08:14:19.368 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-31T08:14:19.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo /usr/sbin/setenforce permissive 2024-08-31T08:14:19.413 DEBUG:teuthology.orchestra.run.smithi167:> sudo /usr/sbin/setenforce permissive 2024-08-31T08:14:19.443 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-31T08:14:19.452 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-08-31T08:14:19.481 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-08-31T08:14:19.600 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-31T08:14:19.611 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-31T08:14:19.612 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi012.front.sepia.ceph.com,smithi167.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-31T08:16:04.976 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi012.front.sepia.ceph.com'), Remote(name='ubuntu@smithi167.front.sepia.ceph.com')] 2024-08-31T08:16:04.977 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-08-31T08:16:04.978 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T08:16:05.063 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-08-31T08:16:05.148 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-08-31T08:16:05.149 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi167.front.sepia.ceph.com' 2024-08-31T08:16:05.149 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi167.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-31T08:16:05.230 DEBUG:teuthology.orchestra.run.smithi167:> true 2024-08-31T08:16:05.266 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi167.front.sepia.ceph.com' 2024-08-31T08:16:05.266 INFO:teuthology.run_tasks:Running task clock... 2024-08-31T08:16:05.277 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-31T08:16:05.277 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-31T08:16:05.277 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T08:16:05.280 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-31T08:16:05.280 DEBUG:teuthology.orchestra.run.smithi167:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-31T08:16:05.307 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-31T08:16:05.324 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-31T08:16:05.334 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-31T08:16:05.346 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: ntpd: command not found 2024-08-31T08:16:05.348 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-31T08:16:05.357 INFO:teuthology.orchestra.run.smithi012.stdout:506 Cannot talk to daemon 2024-08-31T08:16:05.371 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-31T08:16:05.372 INFO:teuthology.orchestra.run.smithi167.stderr:sudo: ntpd: command not found 2024-08-31T08:16:05.383 INFO:teuthology.orchestra.run.smithi167.stdout:506 Cannot talk to daemon 2024-08-31T08:16:05.385 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-31T08:16:05.397 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-31T08:16:05.411 INFO:teuthology.orchestra.run.smithi167.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-31T08:16:05.441 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-08-31T08:16:05.444 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T08:16:05.444 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-08-31T08:16:05.445 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.445 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.445 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.445 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.471 INFO:teuthology.orchestra.run.smithi167.stderr:bash: line 1: ntpq: command not found 2024-08-31T08:16:05.475 INFO:teuthology.orchestra.run.smithi167.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-31T08:16:05.475 INFO:teuthology.orchestra.run.smithi167.stdout:=============================================================================== 2024-08-31T08:16:05.475 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.476 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.476 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.476 INFO:teuthology.orchestra.run.smithi167.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-31T08:16:05.476 INFO:teuthology.run_tasks:Running task install... 2024-08-31T08:16:05.484 DEBUG:teuthology.task.install:project ceph 2024-08-31T08:16:05.484 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}} 2024-08-31T08:16:05.484 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'} 2024-08-31T08:16:05.484 INFO:teuthology.task.install:Using flavor: default 2024-08-31T08:16:05.490 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-31T08:16:05.490 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2024-08-31T08:16:05.490 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'tag': None, 'wait_for_package': False} 2024-08-31T08:16:05.491 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:16:05.493 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'tag': None, 'wait_for_package': False} 2024-08-31T08:16:05.493 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:16:05.658 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/ 2024-08-31T08:16:05.658 INFO:teuthology.task.install.rpm:Package version is 19.3.0-4620.gdc9f1ccb 2024-08-31T08:16:05.670 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/ 2024-08-31T08:16:05.670 INFO:teuthology.task.install.rpm:Package version is 19.3.0-4620.gdc9f1ccb 2024-08-31T08:16:05.780 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-31T08:16:05.780 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:16:05.781 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-31T08:16:05.792 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-dis-testing/dc9f1ccbfcab74acda17f39d6ab018266071e760/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-31T08:16:05.792 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:16:05.792 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-31T08:16:05.807 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-08-31T08:16:05.807 DEBUG:teuthology.orchestra.run.smithi167:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/dc9f1ccbfcab74acda17f39d6ab018266071e760/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-31T08:16:05.818 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2024-08-31T08:16:05.819 DEBUG:teuthology.orchestra.run.smithi012:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/dc9f1ccbfcab74acda17f39d6ab018266071e760/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-31T08:16:05.874 DEBUG:teuthology.orchestra.run.smithi167:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-31T08:16:05.884 DEBUG:teuthology.orchestra.run.smithi012:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-31T08:16:05.951 DEBUG:teuthology.orchestra.run.smithi167:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-31T08:16:05.959 DEBUG:teuthology.orchestra.run.smithi012:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-31T08:16:06.015 INFO:teuthology.orchestra.run.smithi167.stdout:check_obsoletes = 1 2024-08-31T08:16:06.016 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum clean all 2024-08-31T08:16:06.023 INFO:teuthology.orchestra.run.smithi012.stdout:check_obsoletes = 1 2024-08-31T08:16:06.024 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum clean all 2024-08-31T08:16:06.242 INFO:teuthology.orchestra.run.smithi167.stdout:45 files removed 2024-08-31T08:16:06.254 INFO:teuthology.orchestra.run.smithi012.stdout:45 files removed 2024-08-31T08:16:06.269 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-radosgw 2024-08-31T08:16:06.282 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-radosgw 2024-08-31T08:16:06.828 INFO:teuthology.orchestra.run.smithi167.stdout:ceph packages for x86_64 300 kB/s | 84 kB 00:00 2024-08-31T08:16:06.848 INFO:teuthology.orchestra.run.smithi012.stdout:ceph packages for x86_64 296 kB/s | 84 kB 00:00 2024-08-31T08:16:07.112 INFO:teuthology.orchestra.run.smithi167.stdout:ceph noarch packages 100 kB/s | 24 kB 00:00 2024-08-31T08:16:07.148 INFO:teuthology.orchestra.run.smithi012.stdout:ceph noarch packages 94 kB/s | 24 kB 00:00 2024-08-31T08:16:07.388 INFO:teuthology.orchestra.run.smithi167.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-08-31T08:16:07.441 INFO:teuthology.orchestra.run.smithi012.stdout:ceph source packages 7.4 kB/s | 1.9 kB 00:00 2024-08-31T08:16:08.004 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.2 MB 00:00 2024-08-31T08:16:09.307 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-08-31T08:16:11.292 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 24 MB/s | 20 MB 00:00 2024-08-31T08:16:15.667 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.6 MB 00:00 2024-08-31T08:16:17.071 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 20 MB 00:07 2024-08-31T08:16:17.393 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 18 kB 00:00 2024-08-31T08:16:18.280 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 30 MB/s | 22 MB 00:00 2024-08-31T08:16:21.556 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - CRB 9.8 MB/s | 6.6 MB 00:00 2024-08-31T08:16:23.423 INFO:teuthology.orchestra.run.smithi167.stdout:CentOS Stream 9 - Extras packages 29 kB/s | 18 kB 00:00 2024-08-31T08:16:24.019 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 24 kB/s | 1.7 kB 00:00 2024-08-31T08:16:24.310 INFO:teuthology.orchestra.run.smithi167.stdout:Extra Packages for Enterprise Linux 30 MB/s | 22 MB 00:00 2024-08-31T08:16:26.447 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:16:26.449 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 11 M 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrading: 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: librados2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.5 M 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.2 M 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 5.5 M 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 22 M 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 25 k 2024-08-31T08:16:26.450 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 755 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 508 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 5.5 M 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-31T08:16:26.451 INFO:teuthology.orchestra.run.smithi012.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 45 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 143 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 161 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 321 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 306 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 99 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-31T08:16:26.452 INFO:teuthology.orchestra.run.smithi012.stdout:Installing weak dependencies: 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout:Install 30 Packages 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout:Upgrade 2 Packages 2024-08-31T08:16:26.453 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:26.454 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 62 M 2024-08-31T08:16:26.454 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:16:27.195 INFO:teuthology.orchestra.run.smithi012.stdout:(1/32): ceph-base-19.3.0-4620.gdc9f1ccb.el9.x86 8.8 MB/s | 5.5 MB 00:00 2024-08-31T08:16:27.254 INFO:teuthology.orchestra.run.smithi012.stdout:(2/32): ceph-selinux-19.3.0-4620.gdc9f1ccb.el9. 432 kB/s | 25 kB 00:00 2024-08-31T08:16:27.412 INFO:teuthology.orchestra.run.smithi012.stdout:(3/32): ceph-radosgw-19.3.0-4620.gdc9f1ccb.el9. 13 MB/s | 11 MB 00:00 2024-08-31T08:16:27.446 INFO:teuthology.orchestra.run.smithi012.stdout:(4/32): libcephfs2-19.3.0-4620.gdc9f1ccb.el9.x8 3.8 MB/s | 755 kB 00:00 2024-08-31T08:16:27.479 INFO:teuthology.orchestra.run.smithi012.stdout:(5/32): libradosstriper1-19.3.0-4620.gdc9f1ccb. 7.5 MB/s | 508 kB 00:00 2024-08-31T08:16:27.530 INFO:teuthology.orchestra.run.smithi012.stdout:(6/32): python3-ceph-argparse-19.3.0-4620.gdc9f 904 kB/s | 45 kB 00:00 2024-08-31T08:16:27.596 INFO:teuthology.orchestra.run.smithi012.stdout:(7/32): python3-ceph-common-19.3.0-4620.gdc9f1c 2.1 MB/s | 143 kB 00:00 2024-08-31T08:16:27.705 INFO:teuthology.orchestra.run.smithi012.stdout:(8/32): librgw2-19.3.0-4620.gdc9f1ccb.el9.x86_6 21 MB/s | 5.5 MB 00:00 2024-08-31T08:16:27.731 INFO:teuthology.orchestra.run.smithi012.stdout:(9/32): python3-cephfs-19.3.0-4620.gdc9f1ccb.el 1.2 MB/s | 161 kB 00:00 2024-08-31T08:16:27.931 INFO:teuthology.orchestra.run.smithi012.stdout:(10/32): ceph-common-19.3.0-4620.gdc9f1ccb.el9. 16 MB/s | 22 MB 00:01 2024-08-31T08:16:27.956 INFO:teuthology.orchestra.run.smithi012.stdout:(11/32): python3-rados-19.3.0-4620.gdc9f1ccb.el 1.3 MB/s | 321 kB 00:00 2024-08-31T08:16:27.981 INFO:teuthology.orchestra.run.smithi012.stdout:(12/32): python3-rgw-19.3.0-4620.gdc9f1ccb.el9. 1.9 MB/s | 99 kB 00:00 2024-08-31T08:16:28.023 INFO:teuthology.orchestra.run.smithi012.stdout:(13/32): python3-rbd-19.3.0-4620.gdc9f1ccb.el9. 1.0 MB/s | 306 kB 00:00 2024-08-31T08:16:28.157 INFO:teuthology.orchestra.run.smithi012.stdout:(14/32): boost-program-options-1.75.0-8.el9.x86 532 kB/s | 107 kB 00:00 2024-08-31T08:16:28.182 INFO:teuthology.orchestra.run.smithi012.stdout:(15/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 285 kB/s | 45 kB 00:00 2024-08-31T08:16:28.208 INFO:teuthology.orchestra.run.smithi012.stdout:(16/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 708 kB/s | 160 kB 00:00 2024-08-31T08:16:28.258 INFO:teuthology.orchestra.run.smithi012.stdout:(17/32): python3-prettytable-0.7.2-27.el9.noarc 848 kB/s | 42 kB 00:00 2024-08-31T08:16:28.292 INFO:teuthology.orchestra.run.smithi012.stdout:(18/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.6 MB/s | 292 kB 00:00 2024-08-31T08:16:28.325 INFO:teuthology.orchestra.run.smithi012.stdout:(19/32): librdkafka-1.6.1-102.el9.x86_64.rpm 3.9 MB/s | 662 kB 00:00 2024-08-31T08:16:28.350 INFO:teuthology.orchestra.run.smithi012.stdout:(20/32): qatlib-service-24.02.0-1.el9.x86_64.rp 615 kB/s | 36 kB 00:00 2024-08-31T08:16:28.376 INFO:teuthology.orchestra.run.smithi012.stdout:(21/32): qatlib-24.02.0-1.el9.x86_64.rpm 1.8 MB/s | 221 kB 00:00 2024-08-31T08:16:28.401 INFO:teuthology.orchestra.run.smithi012.stdout:(22/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 629 kB/s | 47 kB 00:00 2024-08-31T08:16:28.460 INFO:teuthology.orchestra.run.smithi012.stdout:(23/32): gperftools-libs-2.9.1-2.el9.x86_64.rpm 2.8 MB/s | 309 kB 00:00 2024-08-31T08:16:28.485 INFO:teuthology.orchestra.run.smithi012.stdout:(24/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 386 kB/s | 26 kB 00:00 2024-08-31T08:16:28.562 INFO:teuthology.orchestra.run.smithi012.stdout:(25/32): liboath-2.6.7-2.el9.x86_64.rpm 477 kB/s | 49 kB 00:00 2024-08-31T08:16:28.594 INFO:teuthology.orchestra.run.smithi012.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 620 kB/s | 67 kB 00:00 2024-08-31T08:16:28.636 INFO:teuthology.orchestra.run.smithi012.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 4.5 MB/s | 191 kB 00:00 2024-08-31T08:16:28.669 INFO:teuthology.orchestra.run.smithi012.stdout:(28/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.7 MB/s | 839 kB 00:00 2024-08-31T08:16:28.728 INFO:teuthology.orchestra.run.smithi012.stdout:(29/32): thrift-0.15.0-2.el9.x86_64.rpm 17 MB/s | 1.6 MB 00:00 2024-08-31T08:16:28.812 INFO:teuthology.orchestra.run.smithi012.stdout:(30/32): libarrow-9.0.0-11.el9.x86_64.rpm 10 MB/s | 4.4 MB 00:00 2024-08-31T08:16:28.920 INFO:teuthology.orchestra.run.smithi012.stdout:(31/32): librados2-19.3.0-4620.gdc9f1ccb.el9.x8 14 MB/s | 3.5 MB 00:00 2024-08-31T08:16:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:(32/32): librbd1-19.3.0-4620.gdc9f1ccb.el9.x86_ 12 MB/s | 3.2 MB 00:00 2024-08-31T08:16:29.009 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:16:29.009 INFO:teuthology.orchestra.run.smithi012.stdout:Total 24 MB/s | 62 MB 00:02 2024-08-31T08:16:29.116 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:16:29.155 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:16:29.156 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:16:29.655 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:16:29.656 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:16:29.908 INFO:teuthology.orchestra.run.smithi167.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-08-31T08:16:31.148 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:16:31.229 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-08-31T08:16:31.267 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-08-31T08:16:31.301 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-08-31T08:16:31.323 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-08-31T08:16:31.349 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-08-31T08:16:31.670 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-08-31T08:16:31.729 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-08-31T08:16:32.135 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-08-31T08:16:32.185 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:32.242 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-radosgw x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 11 M 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout:Upgrading: 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout: librados2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.5 M 2024-08-31T08:16:32.245 INFO:teuthology.orchestra.run.smithi167.stdout: librbd1 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.2 M 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-base x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 5.5 M 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-common x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 22 M 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-selinux x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 25 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 755 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-31T08:16:32.246 INFO:teuthology.orchestra.run.smithi167.stdout: libradosstriper1 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 508 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: librgw2 x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 5.5 M 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-argparse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 45 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-common x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 143 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cephfs x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 161 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rados x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 321 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rbd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 306 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rgw x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 99 k 2024-08-31T08:16:32.247 INFO:teuthology.orchestra.run.smithi167.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout:Installing weak dependencies: 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout:Install 30 Packages 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout:Upgrade 2 Packages 2024-08-31T08:16:32.248 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:32.249 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 62 M 2024-08-31T08:16:32.249 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:16:32.278 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:32.368 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 8/34 2024-08-31T08:16:32.395 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 9/34 2024-08-31T08:16:32.458 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 9/34 2024-08-31T08:16:32.487 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-08-31T08:16:32.530 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9. 11/34 2024-08-31T08:16:32.601 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 12/34 2024-08-31T08:16:32.639 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 13/34 2024-08-31T08:16:32.687 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 13/34 2024-08-31T08:16:32.721 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-08-31T08:16:32.833 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-08-31T08:16:32.863 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-08-31T08:16:32.904 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-08-31T08:16:32.929 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 18/34 2024-08-31T08:16:32.959 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-08-31T08:16:32.991 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-08-31T08:16:33.234 INFO:teuthology.orchestra.run.smithi167.stdout:(1/32): ceph-base-19.3.0-4620.gdc9f1ccb.el9.x86 10 MB/s | 5.5 MB 00:00 2024-08-31T08:16:33.301 INFO:teuthology.orchestra.run.smithi167.stdout:(2/32): ceph-selinux-19.3.0-4620.gdc9f1ccb.el9. 378 kB/s | 25 kB 00:00 2024-08-31T08:16:33.328 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-08-31T08:16:33.371 INFO:teuthology.orchestra.run.smithi012.stdout: Upgrading : librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 22/34 2024-08-31T08:16:33.401 INFO:teuthology.orchestra.run.smithi167.stdout:(3/32): libcephfs2-19.3.0-4620.gdc9f1ccb.el9.x8 7.4 MB/s | 755 kB 00:00 2024-08-31T08:16:33.463 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 22/34 2024-08-31T08:16:33.485 INFO:teuthology.orchestra.run.smithi167.stdout:(4/32): libradosstriper1-19.3.0-4620.gdc9f1ccb. 6.0 MB/s | 508 kB 00:00 2024-08-31T08:16:33.492 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 23/34 2024-08-31T08:16:33.528 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-08-31T08:16:33.635 INFO:teuthology.orchestra.run.smithi167.stdout:(5/32): ceph-radosgw-19.3.0-4620.gdc9f1ccb.el9. 11 MB/s | 11 MB 00:00 2024-08-31T08:16:33.686 INFO:teuthology.orchestra.run.smithi167.stdout:(6/32): python3-ceph-argparse-19.3.0-4620.gdc9f 906 kB/s | 45 kB 00:00 2024-08-31T08:16:33.894 INFO:teuthology.orchestra.run.smithi167.stdout:(7/32): ceph-common-19.3.0-4620.gdc9f1ccb.el9.x 18 MB/s | 22 MB 00:01 2024-08-31T08:16:33.920 INFO:teuthology.orchestra.run.smithi167.stdout:(8/32): python3-ceph-common-19.3.0-4620.gdc9f1c 613 kB/s | 143 kB 00:00 2024-08-31T08:16:33.945 INFO:teuthology.orchestra.run.smithi167.stdout:(9/32): python3-cephfs-19.3.0-4620.gdc9f1ccb.el 3.1 MB/s | 161 kB 00:00 2024-08-31T08:16:33.978 INFO:teuthology.orchestra.run.smithi167.stdout:(10/32): python3-rados-19.3.0-4620.gdc9f1ccb.el 5.4 MB/s | 321 kB 00:00 2024-08-31T08:16:34.012 INFO:teuthology.orchestra.run.smithi167.stdout:(11/32): python3-rbd-19.3.0-4620.gdc9f1ccb.el9. 4.5 MB/s | 306 kB 00:00 2024-08-31T08:16:34.037 INFO:teuthology.orchestra.run.smithi167.stdout:(12/32): python3-rgw-19.3.0-4620.gdc9f1ccb.el9. 1.7 MB/s | 99 kB 00:00 2024-08-31T08:16:34.048 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-08-31T08:16:34.082 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 26/34 2024-08-31T08:16:34.121 INFO:teuthology.orchestra.run.smithi167.stdout:(13/32): librgw2-19.3.0-4620.gdc9f1ccb.el9.x86_ 8.7 MB/s | 5.5 MB 00:00 2024-08-31T08:16:34.153 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 26/34 2024-08-31T08:16:34.226 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 27/34 2024-08-31T08:16:34.263 INFO:teuthology.orchestra.run.smithi167.stdout:(14/32): boost-program-options-1.75.0-8.el9.x86 427 kB/s | 107 kB 00:00 2024-08-31T08:16:34.263 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x8 28/34 2024-08-31T08:16:34.288 INFO:teuthology.orchestra.run.smithi167.stdout:(15/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 271 kB/s | 45 kB 00:00 2024-08-31T08:16:34.314 INFO:teuthology.orchestra.run.smithi167.stdout:(16/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 581 kB/s | 160 kB 00:00 2024-08-31T08:16:34.372 INFO:teuthology.orchestra.run.smithi167.stdout:(17/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-08-31T08:16:34.397 INFO:teuthology.orchestra.run.smithi167.stdout:(18/32): python3-prettytable-0.7.2-27.el9.noarc 506 kB/s | 42 kB 00:00 2024-08-31T08:16:34.448 INFO:teuthology.orchestra.run.smithi167.stdout:(19/32): librdkafka-1.6.1-102.el9.x86_64.rpm 3.5 MB/s | 662 kB 00:00 2024-08-31T08:16:34.473 INFO:teuthology.orchestra.run.smithi167.stdout:(20/32): qatlib-service-24.02.0-1.el9.x86_64.rp 477 kB/s | 36 kB 00:00 2024-08-31T08:16:34.506 INFO:teuthology.orchestra.run.smithi167.stdout:(21/32): qatlib-24.02.0-1.el9.x86_64.rpm 1.6 MB/s | 221 kB 00:00 2024-08-31T08:16:34.532 INFO:teuthology.orchestra.run.smithi167.stdout:(22/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 564 kB/s | 47 kB 00:00 2024-08-31T08:16:34.591 INFO:teuthology.orchestra.run.smithi167.stdout:(23/32): gperftools-libs-2.9.1-2.el9.x86_64.rpm 2.6 MB/s | 309 kB 00:00 2024-08-31T08:16:34.633 INFO:teuthology.orchestra.run.smithi167.stdout:(24/32): liboath-2.6.7-2.el9.x86_64.rpm 1.2 MB/s | 49 kB 00:00 2024-08-31T08:16:34.650 INFO:teuthology.orchestra.run.smithi167.stdout:(25/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 241 kB/s | 26 kB 00:00 2024-08-31T08:16:34.675 INFO:teuthology.orchestra.run.smithi167.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 1.6 MB/s | 67 kB 00:00 2024-08-31T08:16:34.700 INFO:teuthology.orchestra.run.smithi167.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 7.5 MB/s | 191 kB 00:00 2024-08-31T08:16:34.750 INFO:teuthology.orchestra.run.smithi167.stdout:(28/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.2 MB/s | 839 kB 00:00 2024-08-31T08:16:34.801 INFO:teuthology.orchestra.run.smithi167.stdout:(29/32): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-08-31T08:16:34.884 INFO:teuthology.orchestra.run.smithi167.stdout:(30/32): libarrow-9.0.0-11.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2024-08-31T08:16:34.993 INFO:teuthology.orchestra.run.smithi167.stdout:(31/32): librados2-19.3.0-4620.gdc9f1ccb.el9.x8 14 MB/s | 3.5 MB 00:00 2024-08-31T08:16:35.068 INFO:teuthology.orchestra.run.smithi167.stdout:(32/32): librbd1-19.3.0-4620.gdc9f1ccb.el9.x86_ 12 MB/s | 3.2 MB 00:00 2024-08-31T08:16:35.074 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:16:35.074 INFO:teuthology.orchestra.run.smithi167.stdout:Total 22 MB/s | 62 MB 00:02 2024-08-31T08:16:35.182 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:16:35.218 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:16:35.218 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:16:35.699 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:16:35.699 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:16:36.801 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:36.838 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:37.191 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:16:37.294 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-08-31T08:16:37.331 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-08-31T08:16:37.357 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-08-31T08:16:37.379 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-08-31T08:16:37.403 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-08-31T08:16:37.423 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:37.467 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 30/34 2024-08-31T08:16:37.559 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 30/34 2024-08-31T08:16:37.560 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-31T08:16:37.560 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-31T08:16:37.560 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:37.595 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:37.700 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-08-31T08:16:37.768 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-08-31T08:16:38.167 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-08-31T08:16:38.234 INFO:teuthology.orchestra.run.smithi167.stdout: Upgrading : librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:38.310 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:38.399 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 8/34 2024-08-31T08:16:38.427 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 9/34 2024-08-31T08:16:38.490 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 9/34 2024-08-31T08:16:38.535 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-08-31T08:16:38.579 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9. 11/34 2024-08-31T08:16:38.649 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 12/34 2024-08-31T08:16:38.671 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 13/34 2024-08-31T08:16:38.735 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 13/34 2024-08-31T08:16:38.769 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-08-31T08:16:38.881 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-08-31T08:16:38.912 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-08-31T08:16:38.952 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-08-31T08:16:38.978 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 18/34 2024-08-31T08:16:39.007 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-08-31T08:16:39.039 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-08-31T08:16:39.360 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-08-31T08:16:39.411 INFO:teuthology.orchestra.run.smithi167.stdout: Upgrading : librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 22/34 2024-08-31T08:16:39.494 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 22/34 2024-08-31T08:16:39.532 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 23/34 2024-08-31T08:16:39.568 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-08-31T08:16:40.062 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-08-31T08:16:40.090 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 26/34 2024-08-31T08:16:40.170 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 26/34 2024-08-31T08:16:40.248 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 27/34 2024-08-31T08:16:40.291 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x8 28/34 2024-08-31T08:16:42.643 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:42.680 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:43.239 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 29/34 2024-08-31T08:16:43.284 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 30/34 2024-08-31T08:16:43.358 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 30/34 2024-08-31T08:16:43.359 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-31T08:16:43.359 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-31T08:16:43.359 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:43.386 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:48.081 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:48.081 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /sys 2024-08-31T08:16:48.081 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /proc 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /mnt 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /var/tmp 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /home 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /root 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout:skipping the directory /tmp 2024-08-31T08:16:48.082 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:49.136 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 32/34 2024-08-31T08:16:49.161 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 32/34 2024-08-31T08:16:49.161 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:16:49.161 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-31T08:16:49.161 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-31T08:16:49.161 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-31T08:16:49.162 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:49.163 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-08-31T08:16:49.246 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-08-31T08:16:49.247 INFO:teuthology.orchestra.run.smithi012.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:50.504 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:50.504 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/34 2024-08-31T08:16:50.504 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 5/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 6/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9. 8/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x8 9/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 10/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 11/34 2024-08-31T08:16:50.505 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 12/34 2024-08-31T08:16:50.506 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 13/34 2024-08-31T08:16:50.506 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-08-31T08:16:50.506 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-08-31T08:16:50.506 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-08-31T08:16:50.506 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 23/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-08-31T08:16:50.507 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-08-31T08:16:50.508 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/34 2024-08-31T08:16:51.287 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout:Upgraded: 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.288 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.289 INFO:teuthology.orchestra.run.smithi012.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: qatlib-24.02.0-1.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-08-31T08:16:51.290 INFO:teuthology.orchestra.run.smithi012.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-08-31T08:16:51.291 INFO:teuthology.orchestra.run.smithi012.stdout: re2-1:20211101-20.el9.x86_64 2024-08-31T08:16:51.291 INFO:teuthology.orchestra.run.smithi012.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-31T08:16:51.291 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:51.291 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:16:51.527 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-test 2024-08-31T08:16:52.102 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:29 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:16:52.245 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 50 M 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 162 k 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:52.246 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout:Install 4 Packages 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 51 M 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 217 M 2024-08-31T08:16:52.247 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:16:52.580 INFO:teuthology.orchestra.run.smithi012.stdout:(1/4): libcephsqlite-19.3.0-4620.gdc9f1ccb.el9. 584 kB/s | 162 kB 00:00 2024-08-31T08:16:53.118 INFO:teuthology.orchestra.run.smithi012.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 373 kB/s | 304 kB 00:00 2024-08-31T08:16:53.228 INFO:teuthology.orchestra.run.smithi012.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 98 kB/s | 64 kB 00:00 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /sys 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /proc 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /mnt 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /var/tmp 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /home 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /root 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout:skipping the directory /tmp 2024-08-31T08:16:53.410 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:54.144 INFO:teuthology.orchestra.run.smithi012.stdout:(4/4): ceph-test-19.3.0-4620.gdc9f1ccb.el9.x86_ 27 MB/s | 50 MB 00:01 2024-08-31T08:16:54.146 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:16:54.146 INFO:teuthology.orchestra.run.smithi012.stdout:Total 27 MB/s | 51 MB 00:01 2024-08-31T08:16:54.174 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:16:54.197 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:16:54.197 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:16:54.414 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 32/34 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 32/34 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-31T08:16:54.446 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:54.448 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-08-31T08:16:54.533 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-08-31T08:16:54.533 INFO:teuthology.orchestra.run.smithi167.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:54.595 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:16:54.595 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:16:55.140 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:16:55.210 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-31T08:16:55.249 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-31T08:16:55.276 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/4 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/34 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/34 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/34 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/34 2024-08-31T08:16:55.846 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 5/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_6 6/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 7/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9. 8/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x8 9/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 10/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 11/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 12/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 13/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-08-31T08:16:55.847 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 23/34 2024-08-31T08:16:55.848 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-08-31T08:16:55.849 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 31/34 2024-08-31T08:16:55.850 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-08-31T08:16:55.850 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/34 2024-08-31T08:16:56.670 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-08-31T08:16:56.670 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:56.670 INFO:teuthology.orchestra.run.smithi167.stdout:Upgraded: 2024-08-31T08:16:56.670 INFO:teuthology.orchestra.run.smithi167.stdout: librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.670 INFO:teuthology.orchestra.run.smithi167.stdout: librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-radosgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-selinux-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-31T08:16:56.671 INFO:teuthology.orchestra.run.smithi167.stdout: libradosstriper1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: librgw2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-argparse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ceph-common-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.672 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: qatlib-24.02.0-1.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: re2-1:20211101-20.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:56.673 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:16:56.965 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-test 2024-08-31T08:16:57.544 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:28 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:16:57.692 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-test x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 50 M 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout: libcephsqlite x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 162 k 2024-08-31T08:16:57.693 INFO:teuthology.orchestra.run.smithi167.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout:Install 4 Packages 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:16:57.694 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 51 M 2024-08-31T08:16:57.695 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 217 M 2024-08-31T08:16:57.695 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:16:57.992 INFO:teuthology.orchestra.run.smithi167.stdout:(1/4): libcephsqlite-19.3.0-4620.gdc9f1ccb.el9. 683 kB/s | 162 kB 00:00 2024-08-31T08:16:58.355 INFO:teuthology.orchestra.run.smithi167.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 176 kB/s | 64 kB 00:00 2024-08-31T08:16:58.405 INFO:teuthology.orchestra.run.smithi167.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 468 kB/s | 304 kB 00:00 2024-08-31T08:16:59.388 INFO:teuthology.orchestra.run.smithi167.stdout:(4/4): ceph-test-19.3.0-4620.gdc9f1ccb.el9.x86_ 31 MB/s | 50 MB 00:01 2024-08-31T08:16:59.390 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:16:59.391 INFO:teuthology.orchestra.run.smithi167.stdout:Total 30 MB/s | 51 MB 00:01 2024-08-31T08:16:59.418 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:16:59.439 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:16:59.439 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:16:59.829 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:16:59.829 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:00.264 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/4 2024-08-31T08:17:00.342 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/4 2024-08-31T08:17:00.370 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:00.448 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-31T08:17:00.497 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-31T08:17:00.525 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/4 2024-08-31T08:17:02.797 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/4 2024-08-31T08:17:02.798 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/4 2024-08-31T08:17:02.798 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/4 2024-08-31T08:17:02.798 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:03.094 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:03.342 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph 2024-08-31T08:17:03.919 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:40 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:04.070 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:04.072 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 6.6 k 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 2.4 M 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 1.5 M 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 291 k 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 4.8 M 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 17 M 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-08-31T08:17:04.073 INFO:teuthology.orchestra.run.smithi012.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-31T08:17:04.074 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-31T08:17:04.075 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout:Install 37 Packages 2024-08-31T08:17:04.076 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:04.077 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 31 M 2024-08-31T08:17:04.077 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 109 M 2024-08-31T08:17:04.077 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:04.733 INFO:teuthology.orchestra.run.smithi012.stdout:(1/37): ceph-19.3.0-4620.gdc9f1ccb.el9.x86_64.r 45 kB/s | 6.6 kB 00:00 2024-08-31T08:17:04.934 INFO:teuthology.orchestra.run.smithi012.stdout:(2/37): ceph-mds-19.3.0-4620.gdc9f1ccb.el9.x86_ 6.9 MB/s | 2.4 MB 00:00 2024-08-31T08:17:04.976 INFO:teuthology.orchestra.run.smithi012.stdout:(3/37): ceph-mgr-19.3.0-4620.gdc9f1ccb.el9.x86_ 3.9 MB/s | 1.5 MB 00:00 2024-08-31T08:17:05.060 INFO:teuthology.orchestra.run.smithi012.stdout:(4/37): ceph-mgr-modules-core-19.3.0-4620.gdc9f 3.4 MB/s | 291 kB 00:00 2024-08-31T08:17:05.176 INFO:teuthology.orchestra.run.smithi012.stdout:(5/37): ceph-mon-19.3.0-4620.gdc9f1ccb.el9.x86_ 11 MB/s | 4.8 MB 00:00 2024-08-31T08:17:05.256 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/4 2024-08-31T08:17:05.340 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/4 2024-08-31T08:17:05.493 INFO:teuthology.orchestra.run.smithi012.stdout:(6/37): ceph-osd-19.3.0-4620.gdc9f1ccb.el9.x86_ 31 MB/s | 17 MB 00:00 2024-08-31T08:17:05.677 INFO:teuthology.orchestra.run.smithi012.stdout:(7/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 411 kB/s | 253 kB 00:00 2024-08-31T08:17:05.785 INFO:teuthology.orchestra.run.smithi012.stdout:(8/37): python3-pycparser-2.20-6.el9.noarch.rpm 1.2 MB/s | 135 kB 00:00 2024-08-31T08:17:05.894 INFO:teuthology.orchestra.run.smithi012.stdout:(9/37): python3-requests-2.25.1-8.el9.noarch.rp 1.1 MB/s | 125 kB 00:00 2024-08-31T08:17:05.919 INFO:teuthology.orchestra.run.smithi012.stdout:(10/37): python3-ply-3.11-14.el9.noarch.rpm 250 kB/s | 106 kB 00:00 2024-08-31T08:17:06.011 INFO:teuthology.orchestra.run.smithi012.stdout:(11/37): python3-urllib3-1.26.5-6.el9.noarch.rp 1.8 MB/s | 215 kB 00:00 2024-08-31T08:17:06.087 INFO:teuthology.orchestra.run.smithi012.stdout:(12/37): lua-5.4.4-4.el9.x86_64.rpm 1.1 MB/s | 188 kB 00:00 2024-08-31T08:17:06.128 INFO:teuthology.orchestra.run.smithi012.stdout:(13/37): python3-cryptography-36.0.1-4.el9.x86_ 1.3 MB/s | 1.2 MB 00:00 2024-08-31T08:17:06.153 INFO:teuthology.orchestra.run.smithi012.stdout:(14/37): python3-markupsafe-1.1.1-12.el9.x86_64 520 kB/s | 35 kB 00:00 2024-08-31T08:17:06.178 INFO:teuthology.orchestra.run.smithi012.stdout:(15/37): python3-mako-1.1.4-6.el9.noarch.rpm 1.0 MB/s | 172 kB 00:00 2024-08-31T08:17:06.204 INFO:teuthology.orchestra.run.smithi012.stdout:(16/37): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-08-31T08:17:06.229 INFO:teuthology.orchestra.run.smithi012.stdout:(17/37): python3-toml-0.10.2-6.el9.noarch.rpm 832 kB/s | 42 kB 00:00 2024-08-31T08:17:06.263 INFO:teuthology.orchestra.run.smithi012.stdout:(18/37): python3-packaging-20.9-5.el9.noarch.rp 575 kB/s | 77 kB 00:00 2024-08-31T08:17:06.305 INFO:teuthology.orchestra.run.smithi012.stdout:(19/37): luarocks-3.9.2-5.el9.noarch.rpm 2.0 MB/s | 151 kB 00:00 2024-08-31T08:17:06.330 INFO:teuthology.orchestra.run.smithi012.stdout:(20/37): lua-devel-5.4.4-4.el9.x86_64.rpm 177 kB/s | 22 kB 00:00 2024-08-31T08:17:06.364 INFO:teuthology.orchestra.run.smithi012.stdout:(21/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 441 kB/s | 43 kB 00:00 2024-08-31T08:17:06.389 INFO:teuthology.orchestra.run.smithi012.stdout:(22/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 464 kB/s | 11 kB 00:00 2024-08-31T08:17:06.414 INFO:teuthology.orchestra.run.smithi012.stdout:(23/37): python3-jaraco-classes-3.2.1-5.el9.noa 715 kB/s | 18 kB 00:00 2024-08-31T08:17:06.439 INFO:teuthology.orchestra.run.smithi012.stdout:(24/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-08-31T08:17:06.465 INFO:teuthology.orchestra.run.smithi012.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 462 kB/s | 23 kB 00:00 2024-08-31T08:17:06.490 INFO:teuthology.orchestra.run.smithi012.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 389 kB/s | 19 kB 00:00 2024-08-31T08:17:06.515 INFO:teuthology.orchestra.run.smithi012.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 393 kB/s | 20 kB 00:00 2024-08-31T08:17:06.540 INFO:teuthology.orchestra.run.smithi012.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 922 kB/s | 46 kB 00:00 2024-08-31T08:17:06.566 INFO:teuthology.orchestra.run.smithi012.stdout:(29/37): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-31T08:17:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:(30/37): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-31T08:17:06.624 INFO:teuthology.orchestra.run.smithi012.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 281 kB/s | 16 kB 00:00 2024-08-31T08:17:06.650 INFO:teuthology.orchestra.run.smithi012.stdout:(32/37): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-08-31T08:17:06.675 INFO:teuthology.orchestra.run.smithi012.stdout:(33/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-08-31T08:17:06.700 INFO:teuthology.orchestra.run.smithi012.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-08-31T08:17:06.734 INFO:teuthology.orchestra.run.smithi012.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 598 kB/s | 20 kB 00:00 2024-08-31T08:17:06.760 INFO:teuthology.orchestra.run.smithi012.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-08-31T08:17:06.793 INFO:teuthology.orchestra.run.smithi012.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.6 MB/s | 427 kB 00:00 2024-08-31T08:17:06.796 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:06.796 INFO:teuthology.orchestra.run.smithi012.stdout:Total 11 MB/s | 31 MB 00:02 2024-08-31T08:17:06.951 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:06.994 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:06.994 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:07.308 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:07.308 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/4 2024-08-31T08:17:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/4 2024-08-31T08:17:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/4 2024-08-31T08:17:07.946 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-31T08:17:08.254 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-31T08:17:08.254 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-test-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout: libcephsqlite-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:08.255 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:08.577 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph 2024-08-31T08:17:08.680 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:08.746 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-08-31T08:17:08.778 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-08-31T08:17:08.806 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-08-31T08:17:08.873 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-08-31T08:17:08.914 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-08-31T08:17:08.946 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-08-31T08:17:09.050 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-08-31T08:17:09.136 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-08-31T08:17:09.141 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:40 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:09.183 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-08-31T08:17:09.218 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-08-31T08:17:09.247 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-08-31T08:17:09.282 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-08-31T08:17:09.294 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:09.297 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:09.297 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:09.297 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:09.297 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 6.6 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mds x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 2.4 M 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 1.5 M 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-modules-core noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 291 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mon x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 4.8 M 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-osd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 17 M 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-31T08:17:09.298 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-collections 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-31T08:17:09.299 INFO:teuthology.orchestra.run.smithi167.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-31T08:17:09.300 INFO:teuthology.orchestra.run.smithi167.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout:Install 37 Packages 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 31 M 2024-08-31T08:17:09.301 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 109 M 2024-08-31T08:17:09.302 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:09.315 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-08-31T08:17:09.356 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-08-31T08:17:09.391 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-08-31T08:17:09.425 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-08-31T08:17:09.474 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-08-31T08:17:09.514 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-08-31T08:17:09.590 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-08-31T08:17:09.732 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-08-31T08:17:09.816 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-08-31T08:17:09.868 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-08-31T08:17:09.905 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-08-31T08:17:09.940 INFO:teuthology.orchestra.run.smithi167.stdout:(1/37): ceph-19.3.0-4620.gdc9f1ccb.el9.x86_64.r 43 kB/s | 6.6 kB 00:00 2024-08-31T08:17:09.955 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-08-31T08:17:10.019 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-08-31T08:17:10.115 INFO:teuthology.orchestra.run.smithi167.stdout:(2/37): ceph-mgr-19.3.0-4620.gdc9f1ccb.el9.x86_ 4.7 MB/s | 1.5 MB 00:00 2024-08-31T08:17:10.166 INFO:teuthology.orchestra.run.smithi167.stdout:(3/37): ceph-mds-19.3.0-4620.gdc9f1ccb.el9.x86_ 6.3 MB/s | 2.4 MB 00:00 2024-08-31T08:17:10.212 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-08-31T08:17:10.241 INFO:teuthology.orchestra.run.smithi167.stdout:(4/37): ceph-mgr-modules-core-19.3.0-4620.gdc9f 3.8 MB/s | 291 kB 00:00 2024-08-31T08:17:10.259 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-08-31T08:17:10.339 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-08-31T08:17:10.429 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-08-31T08:17:10.466 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-08-31T08:17:10.467 INFO:teuthology.orchestra.run.smithi167.stdout:(5/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-08-31T08:17:10.601 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-08-31T08:17:10.667 INFO:teuthology.orchestra.run.smithi167.stdout:(6/37): ceph-osd-19.3.0-4620.gdc9f1ccb.el9.x86_ 31 MB/s | 17 MB 00:00 2024-08-31T08:17:10.750 INFO:teuthology.orchestra.run.smithi167.stdout:(7/37): ceph-mon-19.3.0-4620.gdc9f1ccb.el9.x86_ 5.9 MB/s | 4.8 MB 00:00 2024-08-31T08:17:10.771 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9. 32/37 2024-08-31T08:17:10.802 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/37 2024-08-31T08:17:10.828 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/37 2024-08-31T08:17:10.828 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:10.828 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-31T08:17:10.828 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-31T08:17:10.828 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-31T08:17:10.829 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:10.834 INFO:teuthology.orchestra.run.smithi167.stdout:(8/37): python3-cryptography-36.0.1-4.el9.x86_6 3.3 MB/s | 1.2 MB 00:00 2024-08-31T08:17:10.893 INFO:teuthology.orchestra.run.smithi167.stdout:(9/37): python3-requests-2.25.1-8.el9.noarch.rp 2.1 MB/s | 125 kB 00:00 2024-08-31T08:17:10.926 INFO:teuthology.orchestra.run.smithi167.stdout:(10/37): python3-ply-3.11-14.el9.noarch.rpm 411 kB/s | 106 kB 00:00 2024-08-31T08:17:10.952 INFO:teuthology.orchestra.run.smithi167.stdout:(11/37): python3-pycparser-2.20-6.el9.noarch.rp 672 kB/s | 135 kB 00:00 2024-08-31T08:17:11.035 INFO:teuthology.orchestra.run.smithi167.stdout:(12/37): python3-urllib3-1.26.5-6.el9.noarch.rp 1.5 MB/s | 215 kB 00:00 2024-08-31T08:17:11.061 INFO:teuthology.orchestra.run.smithi167.stdout:(13/37): python3-mako-1.1.4-6.el9.noarch.rpm 1.6 MB/s | 172 kB 00:00 2024-08-31T08:17:11.086 INFO:teuthology.orchestra.run.smithi167.stdout:(14/37): lua-5.4.4-4.el9.x86_64.rpm 1.2 MB/s | 188 kB 00:00 2024-08-31T08:17:11.111 INFO:teuthology.orchestra.run.smithi167.stdout:(15/37): python3-packaging-20.9-5.el9.noarch.rp 1.5 MB/s | 77 kB 00:00 2024-08-31T08:17:11.136 INFO:teuthology.orchestra.run.smithi167.stdout:(16/37): python3-markupsafe-1.1.1-12.el9.x86_64 345 kB/s | 35 kB 00:00 2024-08-31T08:17:11.162 INFO:teuthology.orchestra.run.smithi167.stdout:(17/37): python3-pytz-2021.1-5.el9.noarch.rpm 674 kB/s | 51 kB 00:00 2024-08-31T08:17:11.187 INFO:teuthology.orchestra.run.smithi167.stdout:(18/37): python3-toml-0.10.2-6.el9.noarch.rpm 551 kB/s | 42 kB 00:00 2024-08-31T08:17:11.212 INFO:teuthology.orchestra.run.smithi167.stdout:(19/37): lua-devel-5.4.4-4.el9.x86_64.rpm 295 kB/s | 22 kB 00:00 2024-08-31T08:17:11.263 INFO:teuthology.orchestra.run.smithi167.stdout:(20/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 577 kB/s | 43 kB 00:00 2024-08-31T08:17:11.288 INFO:teuthology.orchestra.run.smithi167.stdout:(21/37): luarocks-3.9.2-5.el9.noarch.rpm 1.2 MB/s | 151 kB 00:00 2024-08-31T08:17:11.314 INFO:teuthology.orchestra.run.smithi167.stdout:(22/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 432 kB/s | 11 kB 00:00 2024-08-31T08:17:11.339 INFO:teuthology.orchestra.run.smithi167.stdout:(23/37): python3-jaraco-classes-3.2.1-5.el9.noa 706 kB/s | 18 kB 00:00 2024-08-31T08:17:11.364 INFO:teuthology.orchestra.run.smithi167.stdout:(24/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.2 MB/s | 172 kB 00:00 2024-08-31T08:17:11.389 INFO:teuthology.orchestra.run.smithi167.stdout:(25/37): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-08-31T08:17:11.415 INFO:teuthology.orchestra.run.smithi167.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-08-31T08:17:11.440 INFO:teuthology.orchestra.run.smithi167.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 391 kB/s | 20 kB 00:00 2024-08-31T08:17:11.465 INFO:teuthology.orchestra.run.smithi167.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-08-31T08:17:11.490 INFO:teuthology.orchestra.run.smithi167.stdout:(29/37): python3-cherrypy-18.6.1-2.el9.noarch.r 1.5 MB/s | 358 kB 00:00 2024-08-31T08:17:11.524 INFO:teuthology.orchestra.run.smithi167.stdout:(30/37): python3-more-itertools-8.12.0-2.el9.no 941 kB/s | 79 kB 00:00 2024-08-31T08:17:11.549 INFO:teuthology.orchestra.run.smithi167.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 280 kB/s | 16 kB 00:00 2024-08-31T08:17:11.575 INFO:teuthology.orchestra.run.smithi167.stdout:(32/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-08-31T08:17:11.592 INFO:teuthology.orchestra.run.smithi167.stdout:(33/37): python3-tempora-5.0.0-2.el9.noarch.rpm 851 kB/s | 36 kB 00:00 2024-08-31T08:17:11.617 INFO:teuthology.orchestra.run.smithi167.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-08-31T08:17:11.634 INFO:teuthology.orchestra.run.smithi167.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 1.2 MB/s | 20 kB 00:00 2024-08-31T08:17:11.659 INFO:teuthology.orchestra.run.smithi167.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-08-31T08:17:11.685 INFO:teuthology.orchestra.run.smithi167.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.5 MB/s | 427 kB 00:00 2024-08-31T08:17:11.689 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:11.689 INFO:teuthology.orchestra.run.smithi167.stdout:Total 13 MB/s | 31 MB 00:02 2024-08-31T08:17:11.837 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:11.877 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:11.877 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:12.176 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:12.177 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:12.369 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 34/37 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 34/37 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-31T08:17:12.396 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:12.861 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 35/37 2024-08-31T08:17:12.884 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 35/37 2024-08-31T08:17:12.885 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:12.885 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-31T08:17:12.885 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-31T08:17:12.885 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-31T08:17:12.885 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:13.120 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 36/37 2024-08-31T08:17:13.352 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 36/37 2024-08-31T08:17:13.352 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:13.353 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-31T08:17:13.353 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-31T08:17:13.353 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-31T08:17:13.353 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:13.353 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 37/37 2024-08-31T08:17:13.529 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:13.596 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-08-31T08:17:13.629 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-08-31T08:17:13.656 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-08-31T08:17:13.720 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-08-31T08:17:13.756 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-08-31T08:17:13.789 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-08-31T08:17:13.882 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-08-31T08:17:13.959 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-08-31T08:17:14.000 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-08-31T08:17:14.035 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-08-31T08:17:14.065 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-08-31T08:17:14.100 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-08-31T08:17:14.133 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-08-31T08:17:14.173 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-08-31T08:17:14.209 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-08-31T08:17:14.244 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-08-31T08:17:14.292 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-08-31T08:17:14.324 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-08-31T08:17:14.406 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-08-31T08:17:14.544 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-08-31T08:17:14.624 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-08-31T08:17:14.685 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-08-31T08:17:14.724 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-08-31T08:17:14.773 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-08-31T08:17:14.829 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 37/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/37 2024-08-31T08:17:14.871 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 5/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9. 6/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-08-31T08:17:14.872 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-08-31T08:17:14.873 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-08-31T08:17:14.874 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-08-31T08:17:14.874 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-08-31T08:17:14.875 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-08-31T08:17:15.015 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-08-31T08:17:15.062 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-08-31T08:17:15.140 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-08-31T08:17:15.229 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-08-31T08:17:15.269 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-08-31T08:17:15.397 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-08-31T08:17:15.559 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9. 32/37 2024-08-31T08:17:15.597 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/37 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 33/37 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-31T08:17:15.624 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:15.907 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: lua-5.4.4-4.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: luarocks-3.9.2-5.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-31T08:17:15.908 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-ply-3.11-14.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-31T08:17:15.909 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-webob-1.8.8-2.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:15.910 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:16.261 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-base 2024-08-31T08:17:16.832 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:53 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:16.925 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:17:16.968 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:16.969 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:17:16.970 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:17.033 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install cephadm 2024-08-31T08:17:17.090 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 34/37 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 34/37 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-31T08:17:17.117 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:17.566 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 35/37 2024-08-31T08:17:17.589 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 35/37 2024-08-31T08:17:17.589 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:17.589 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-31T08:17:17.590 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-31T08:17:17.590 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-31T08:17:17.590 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:17.593 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:54 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:17.729 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:17.730 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 783 k 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:17.731 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:17.732 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:17.732 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:17:17.732 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:17.732 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 783 k 2024-08-31T08:17:17.732 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 789 k 2024-08-31T08:17:17.733 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:17.817 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 36/37 2024-08-31T08:17:17.840 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 36/37 2024-08-31T08:17:17.840 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:17.840 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-31T08:17:17.841 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-31T08:17:17.841 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-31T08:17:17.841 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:17.903 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 37/37 2024-08-31T08:17:18.051 INFO:teuthology.orchestra.run.smithi012.stdout:cephadm-19.3.0-4620.gdc9f1ccb.el9.noarch.rpm 2.4 MB/s | 783 kB 00:00 2024-08-31T08:17:18.051 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:18.052 INFO:teuthology.orchestra.run.smithi012.stdout:Total 2.4 MB/s | 783 kB 00:00 2024-08-31T08:17:18.052 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:18.057 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:18.057 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:18.063 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:18.064 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:18.140 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:18.585 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:18.673 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:19.209 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:19.460 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:19.460 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:19.461 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:19.461 INFO:teuthology.orchestra.run.smithi012.stdout: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:19.461 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:19.461 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:19.650 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-immutable-object-cache 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 37/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 3/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 4/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 5/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9. 6/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-08-31T08:17:19.689 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-08-31T08:17:19.690 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-08-31T08:17:19.691 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-08-31T08:17:20.219 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:57 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:20.354 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 145 k 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 145 k 2024-08-31T08:17:20.355 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 432 k 2024-08-31T08:17:20.356 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:20.577 INFO:teuthology.orchestra.run.smithi012.stdout:ceph-immutable-object-cache-19.3.0-4620.gdc9f1c 656 kB/s | 145 kB 00:00 2024-08-31T08:17:20.577 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:20.578 INFO:teuthology.orchestra.run.smithi012.stdout:Total 651 kB/s | 145 kB 00:00 2024-08-31T08:17:20.578 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:20.586 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:20.586 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:20.644 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:20.645 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mds-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-modules-core-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:20.778 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mon-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-osd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: lua-5.4.4-4.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: luarocks-3.9.2-5.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-31T08:17:20.779 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-ply-3.11-14.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-31T08:17:20.780 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-webob-1.8.8-2.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:20.781 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:20.803 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:20.878 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:20.903 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:20.903 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:20.903 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-31T08:17:20.903 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:21.136 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-base 2024-08-31T08:17:21.701 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:52 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:21.791 INFO:teuthology.orchestra.run.smithi167.stdout:Package ceph-base-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:17:21.800 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:21.800 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:21.800 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:21.800 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:21.800 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:21.801 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:21.833 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:21.834 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:17:21.834 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:21.907 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install cephadm 2024-08-31T08:17:22.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr 2024-08-31T08:17:22.464 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:53 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:22.596 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:22.597 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:22.597 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:22.597 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:22.597 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout: cephadm noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 783 k 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 783 k 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 789 k 2024-08-31T08:17:22.598 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:22.601 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:00:59 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:22.694 INFO:teuthology.orchestra.run.smithi012.stdout:Package ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:17:22.738 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:22.739 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:17:22.739 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:22.811 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-dashboard 2024-08-31T08:17:22.867 INFO:teuthology.orchestra.run.smithi167.stdout:cephadm-19.3.0-4620.gdc9f1ccb.el9.noarch.rpm 2.8 MB/s | 783 kB 00:00 2024-08-31T08:17:22.868 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:22.868 INFO:teuthology.orchestra.run.smithi167.stdout:Total 2.8 MB/s | 783 kB 00:00 2024-08-31T08:17:22.868 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:22.873 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:22.873 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:22.879 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:22.880 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:22.955 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:23.354 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:23.370 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:00 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:23.440 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:23.507 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:23.508 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:23.508 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:23.508 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:23.508 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 14 M 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 34 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 16 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-08-31T08:17:23.509 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout:Install 12 Packages 2024-08-31T08:17:23.510 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:23.511 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 19 M 2024-08-31T08:17:23.511 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 158 M 2024-08-31T08:17:23.511 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:23.785 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:24.029 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:24.029 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:24.030 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:24.030 INFO:teuthology.orchestra.run.smithi167.stdout: cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:24.030 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:24.030 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:24.259 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-immutable-object-cache 2024-08-31T08:17:24.497 INFO:teuthology.orchestra.run.smithi012.stdout:(1/12): ceph-prometheus-alerts-19.3.0-4620.gdc9 106 kB/s | 16 kB 00:00 2024-08-31T08:17:24.522 INFO:teuthology.orchestra.run.smithi012.stdout:(2/12): ceph-grafana-dashboards-19.3.0-4620.gdc 189 kB/s | 34 kB 00:00 2024-08-31T08:17:24.731 INFO:teuthology.orchestra.run.smithi012.stdout:(3/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.3 MB/s | 269 kB 00:00 2024-08-31T08:17:24.827 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:55 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:24.961 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:24.961 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-immutable-object-cache x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 145 k 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 145 k 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 432 k 2024-08-31T08:17:24.962 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:24.973 INFO:teuthology.orchestra.run.smithi012.stdout:(4/12): ceph-mgr-dashboard-19.3.0-4620.gdc9f1cc 22 MB/s | 14 MB 00:00 2024-08-31T08:17:25.006 INFO:teuthology.orchestra.run.smithi012.stdout:(5/12): protobuf-compiler-3.14.0-13.el9.x86_64. 3.1 MB/s | 863 kB 00:00 2024-08-31T08:17:25.048 INFO:teuthology.orchestra.run.smithi012.stdout:(6/12): protobuf-3.14.0-13.el9.x86_64.rpm 1.8 MB/s | 1.0 MB 00:00 2024-08-31T08:17:25.099 INFO:teuthology.orchestra.run.smithi012.stdout:(7/12): grpc-data-1.46.7-10.el9.noarch.rpm 231 kB/s | 19 kB 00:00 2024-08-31T08:17:25.124 INFO:teuthology.orchestra.run.smithi012.stdout:(8/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 6.2 MB/s | 144 kB 00:00 2024-08-31T08:17:25.149 INFO:teuthology.orchestra.run.smithi012.stdout:(9/12): python3-repoze-lru-0.7-16.el9.noarch.rp 1.2 MB/s | 31 kB 00:00 2024-08-31T08:17:25.175 INFO:teuthology.orchestra.run.smithi012.stdout:(10/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 2.7 MB/s | 548 kB 00:00 2024-08-31T08:17:25.186 INFO:teuthology.orchestra.run.smithi167.stdout:ceph-immutable-object-cache-19.3.0-4620.gdc9f1c 649 kB/s | 145 kB 00:00 2024-08-31T08:17:25.186 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:25.187 INFO:teuthology.orchestra.run.smithi167.stdout:Total 645 kB/s | 145 kB 00:00 2024-08-31T08:17:25.187 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:25.194 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:25.195 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:25.208 INFO:teuthology.orchestra.run.smithi012.stdout:(11/12): python3-routes-2.5.1-5.el9.noarch.rpm 3.1 MB/s | 188 kB 00:00 2024-08-31T08:17:25.252 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:25.252 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:25.267 INFO:teuthology.orchestra.run.smithi012.stdout:(12/12): python3-grpcio-1.46.7-10.el9.x86_64.rp 9.8 MB/s | 2.0 MB 00:00 2024-08-31T08:17:25.272 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:25.272 INFO:teuthology.orchestra.run.smithi012.stdout:Total 11 MB/s | 19 MB 00:01 2024-08-31T08:17:25.324 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:25.344 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:25.344 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:25.391 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:25.479 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:25.504 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:25.504 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:25.504 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-31T08:17:25.504 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:25.578 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:25.578 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:26.178 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:26.252 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-08-31T08:17:26.295 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-08-31T08:17:26.342 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-08-31T08:17:26.407 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb. 1/1 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-immutable-object-cache-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:26.414 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:26.509 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-08-31T08:17:26.636 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr 2024-08-31T08:17:26.644 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-08-31T08:17:26.745 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-08-31T08:17:26.794 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-08-31T08:17:26.821 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-08-31T08:17:26.854 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9 10/12 2024-08-31T08:17:27.205 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:58 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:27.298 INFO:teuthology.orchestra.run.smithi167.stdout:Package ceph-mgr-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:17:27.341 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:27.343 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:17:27.343 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:27.415 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-dashboard 2024-08-31T08:17:27.967 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:00:58 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:28.099 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-dashboard noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 14 M 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:17:28.100 INFO:teuthology.orchestra.run.smithi167.stdout: abseil-cpp x86_64 20211102.0-3.el9 epel 548 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-grafana-dashboards noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 34 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-prometheus-alerts noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 16 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: protobuf x86_64 3.14.0-13.el9 appstream 1.0 M 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: protobuf-compiler x86_64 3.14.0-13.el9 crb 863 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: python3-protobuf noarch 3.14.0-13.el9 appstream 269 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:28.101 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout:Install 12 Packages 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 19 M 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 158 M 2024-08-31T08:17:28.102 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:28.681 INFO:teuthology.orchestra.run.smithi167.stdout:(1/12): ceph-grafana-dashboards-19.3.0-4620.gdc 165 kB/s | 34 kB 00:00 2024-08-31T08:17:28.723 INFO:teuthology.orchestra.run.smithi167.stdout:(2/12): ceph-prometheus-alerts-19.3.0-4620.gdc9 66 kB/s | 16 kB 00:00 2024-08-31T08:17:28.948 INFO:teuthology.orchestra.run.smithi167.stdout:(3/12): python3-protobuf-3.14.0-13.el9.noarch.r 1.2 MB/s | 269 kB 00:00 2024-08-31T08:17:28.990 INFO:teuthology.orchestra.run.smithi167.stdout:(4/12): protobuf-3.14.0-13.el9.x86_64.rpm 3.3 MB/s | 1.0 MB 00:00 2024-08-31T08:17:29.040 INFO:teuthology.orchestra.run.smithi167.stdout:(5/12): abseil-cpp-20211102.0-3.el9.x86_64.rpm 11 MB/s | 548 kB 00:00 2024-08-31T08:17:29.065 INFO:teuthology.orchestra.run.smithi167.stdout:(6/12): grpc-data-1.46.7-10.el9.noarch.rpm 786 kB/s | 19 kB 00:00 2024-08-31T08:17:29.133 INFO:teuthology.orchestra.run.smithi167.stdout:(7/12): python3-grpcio-1.46.7-10.el9.x86_64.rpm 30 MB/s | 2.0 MB 00:00 2024-08-31T08:17:29.158 INFO:teuthology.orchestra.run.smithi167.stdout:(8/12): python3-grpcio-tools-1.46.7-10.el9.x86_ 5.6 MB/s | 144 kB 00:00 2024-08-31T08:17:29.184 INFO:teuthology.orchestra.run.smithi167.stdout:(9/12): python3-repoze-lru-0.7-16.el9.noarch.rp 1.2 MB/s | 31 kB 00:00 2024-08-31T08:17:29.209 INFO:teuthology.orchestra.run.smithi167.stdout:(10/12): python3-routes-2.5.1-5.el9.noarch.rpm 7.4 MB/s | 188 kB 00:00 2024-08-31T08:17:29.284 INFO:teuthology.orchestra.run.smithi167.stdout:(11/12): protobuf-compiler-3.14.0-13.el9.x86_64 2.5 MB/s | 863 kB 00:00 2024-08-31T08:17:30.292 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el 11/12 2024-08-31T08:17:30.349 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 12/12 2024-08-31T08:17:30.426 INFO:teuthology.orchestra.run.smithi167.stdout:(12/12): ceph-mgr-dashboard-19.3.0-4620.gdc9f1c 7.2 MB/s | 14 MB 00:01 2024-08-31T08:17:30.430 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:30.430 INFO:teuthology.orchestra.run.smithi167.stdout:Total 8.3 MB/s | 19 MB 00:02 2024-08-31T08:17:30.482 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:30.502 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:30.502 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:30.730 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:30.730 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:31.317 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:31.393 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : protobuf-3.14.0-13.el9.x86_64 1/12 2024-08-31T08:17:31.428 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : protobuf-compiler-3.14.0-13.el9.x86_64 2/12 2024-08-31T08:17:31.475 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/12 2024-08-31T08:17:31.505 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/12 2024-08-31T08:17:31.590 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 5/12 2024-08-31T08:17:31.724 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : abseil-cpp-20211102.0-3.el9.x86_64 6/12 2024-08-31T08:17:31.843 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 7/12 2024-08-31T08:17:31.894 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-protobuf-3.14.0-13.el9.noarch 8/12 2024-08-31T08:17:31.925 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 9/12 2024-08-31T08:17:31.955 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9 10/12 2024-08-31T08:17:32.289 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 12/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el 1/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 2/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9 3/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-08-31T08:17:32.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-08-31T08:17:32.599 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-08-31T08:17:32.599 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: grpc-data-1.46.7-10.el9.noarch 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: protobuf-3.14.0-13.el9.x86_64 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-08-31T08:17:32.600 INFO:teuthology.orchestra.run.smithi012.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-08-31T08:17:32.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-31T08:17:32.601 INFO:teuthology.orchestra.run.smithi012.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-31T08:17:32.601 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:32.601 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:32.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-31T08:17:33.396 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:10 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:33.535 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================= 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================= 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 7.4 M 2024-08-31T08:17:33.536 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-31T08:17:33.537 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout:======================================================================================= 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout:Install 12 Packages 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:33.538 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 38 M 2024-08-31T08:17:33.539 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 212 M 2024-08-31T08:17:33.539 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:33.816 INFO:teuthology.orchestra.run.smithi012.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.3 MB/s | 191 kB 00:00 2024-08-31T08:17:33.866 INFO:teuthology.orchestra.run.smithi012.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 4.0 MB/s | 801 kB 00:00 2024-08-31T08:17:33.949 INFO:teuthology.orchestra.run.smithi012.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 245 kB/s | 33 kB 00:00 2024-08-31T08:17:34.000 INFO:teuthology.orchestra.run.smithi012.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 364 kB/s | 18 kB 00:00 2024-08-31T08:17:34.067 INFO:teuthology.orchestra.run.smithi012.stdout:(5/12): openblas-0.3.26-2.el9.x86_64.rpm 576 kB/s | 38 kB 00:00 2024-08-31T08:17:34.167 INFO:teuthology.orchestra.run.smithi012.stdout:(6/12): ceph-mgr-diskprediction-local-19.3.0-46 15 MB/s | 7.4 MB 00:00 2024-08-31T08:17:34.234 INFO:teuthology.orchestra.run.smithi012.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 8.1 MB/s | 3.0 MB 00:00 2024-08-31T08:17:34.368 INFO:teuthology.orchestra.run.smithi012.stdout:(8/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 38 MB/s | 5.1 MB 00:00 2024-08-31T08:17:34.444 INFO:teuthology.orchestra.run.smithi012.stdout:(9/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 13 MB/s | 4.9 MB 00:00 2024-08-31T08:17:34.469 INFO:teuthology.orchestra.run.smithi012.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.4 MB/s | 250 kB 00:00 2024-08-31T08:17:34.502 INFO:teuthology.orchestra.run.smithi012.stdout:(11/12): python3-devel-3.9.19-8.el9.x86_64.rpm 731 kB/s | 245 kB 00:00 2024-08-31T08:17:34.761 INFO:teuthology.orchestra.run.smithi012.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 51 MB/s | 16 MB 00:00 2024-08-31T08:17:34.761 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:34.762 INFO:teuthology.orchestra.run.smithi012.stdout:Total 31 MB/s | 38 MB 00:01 2024-08-31T08:17:35.127 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:35.130 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el 11/12 2024-08-31T08:17:35.154 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:35.155 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:35.185 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 12/12 2024-08-31T08:17:35.590 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:35.591 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:36.502 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:36.557 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-31T08:17:36.593 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-08-31T08:17:36.699 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-08-31T08:17:36.735 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-08-31T08:17:36.926 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-08-31T08:17:36.958 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-08-31T08:17:37.037 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-31T08:17:37.078 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 12/12 2024-08-31T08:17:37.078 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el 1/12 2024-08-31T08:17:37.078 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noa 2/12 2024-08-31T08:17:37.078 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9 3/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : protobuf-3.14.0-13.el9.x86_64 4/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-protobuf-3.14.0-13.el9.noarch 5/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : protobuf-compiler-3.14.0-13.el9.x86_64 6/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : abseil-cpp-20211102.0-3.el9.x86_64 7/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 8/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 9/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/12 2024-08-31T08:17:37.079 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 11/12 2024-08-31T08:17:37.380 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 12/12 2024-08-31T08:17:37.380 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:37.380 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:37.380 INFO:teuthology.orchestra.run.smithi167.stdout: abseil-cpp-20211102.0-3.el9.x86_64 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-grafana-dashboards-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-dashboard-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-prometheus-alerts-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: grpc-data-1.46.7-10.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: protobuf-3.14.0-13.el9.x86_64 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: protobuf-compiler-3.14.0-13.el9.x86_64 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: python3-protobuf-3.14.0-13.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-31T08:17:37.381 INFO:teuthology.orchestra.run.smithi167.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-31T08:17:37.382 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:37.382 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:37.634 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-31T08:17:37.729 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-31T08:17:37.843 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-31T08:17:38.191 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:09 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:38.325 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:38.326 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================= 2024-08-31T08:17:38.326 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:38.326 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================= 2024-08-31T08:17:38.326 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:38.326 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-diskprediction-local noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 7.4 M 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:38.327 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout:======================================================================================= 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout:Install 12 Packages 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 38 M 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 212 M 2024-08-31T08:17:38.328 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:38.935 INFO:teuthology.orchestra.run.smithi167.stdout:(1/12): ceph-mgr-diskprediction-local-19.3.0-46 16 MB/s | 7.4 MB 00:00 2024-08-31T08:17:39.142 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-31T08:17:39.302 INFO:teuthology.orchestra.run.smithi167.stdout:(2/12): libquadmath-11.5.0-2.el9.x86_64.rpm 234 kB/s | 191 kB 00:00 2024-08-31T08:17:39.510 INFO:teuthology.orchestra.run.smithi167.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 782 kB/s | 801 kB 00:01 2024-08-31T08:17:39.527 INFO:teuthology.orchestra.run.smithi167.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 55 kB/s | 33 kB 00:00 2024-08-31T08:17:39.985 INFO:teuthology.orchestra.run.smithi167.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 38 kB/s | 18 kB 00:00 2024-08-31T08:17:40.087 INFO:teuthology.orchestra.run.smithi167.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 70 kB/s | 38 kB 00:00 2024-08-31T08:17:40.218 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-31T08:17:40.249 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 12/12 2024-08-31T08:17:40.353 INFO:teuthology.orchestra.run.smithi167.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.8 MB/s | 3.0 MB 00:01 2024-08-31T08:17:40.853 INFO:teuthology.orchestra.run.smithi167.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 319 kB/s | 245 kB 00:00 2024-08-31T08:17:41.103 INFO:teuthology.orchestra.run.smithi167.stdout:(9/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 4.4 MB/s | 4.9 MB 00:01 2024-08-31T08:17:41.645 INFO:teuthology.orchestra.run.smithi167.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 3.9 MB/s | 5.1 MB 00:01 2024-08-31T08:17:41.670 INFO:teuthology.orchestra.run.smithi167.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 307 kB/s | 250 kB 00:00 2024-08-31T08:17:42.604 INFO:teuthology.orchestra.run.smithi167.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-08-31T08:17:42.605 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:42.605 INFO:teuthology.orchestra.run.smithi167.stdout:Total 8.9 MB/s | 38 MB 00:04 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 12/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 1/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-31T08:17:42.776 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-08-31T08:17:42.777 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-08-31T08:17:42.777 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-08-31T08:17:42.777 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-31T08:17:42.777 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-31T08:17:42.968 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:42.995 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:42.996 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:43.189 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-0.3.26-2.el9.x86_64 2024-08-31T08:17:43.190 INFO:teuthology.orchestra.run.smithi012.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:43.191 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:43.416 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:43.416 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:43.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-rook 2024-08-31T08:17:44.123 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:21 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:44.185 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:44.258 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-31T08:17:44.261 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:44.262 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:44.262 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 49 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-31T08:17:44.263 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout:Install 13 Packages 2024-08-31T08:17:44.264 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:44.265 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 2.2 M 2024-08-31T08:17:44.265 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 26 M 2024-08-31T08:17:44.265 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:44.294 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-08-31T08:17:44.394 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-08-31T08:17:44.445 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-08-31T08:17:44.630 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-08-31T08:17:44.669 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-08-31T08:17:44.745 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-31T08:17:45.073 INFO:teuthology.orchestra.run.smithi012.stdout:(1/13): ceph-mgr-rook-19.3.0-4620.gdc9f1ccb.el9 217 kB/s | 49 kB 00:00 2024-08-31T08:17:45.107 INFO:teuthology.orchestra.run.smithi012.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 73 kB/s | 19 kB 00:00 2024-08-31T08:17:45.132 INFO:teuthology.orchestra.run.smithi012.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 89 kB/s | 26 kB 00:00 2024-08-31T08:17:45.394 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-31T08:17:45.465 INFO:teuthology.orchestra.run.smithi012.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 443 kB/s | 159 kB 00:00 2024-08-31T08:17:45.507 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-31T08:17:45.557 INFO:teuthology.orchestra.run.smithi012.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 656 kB/s | 279 kB 00:00 2024-08-31T08:17:45.591 INFO:teuthology.orchestra.run.smithi012.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 427 kB/s | 54 kB 00:00 2024-08-31T08:17:45.616 INFO:teuthology.orchestra.run.smithi012.stdout:(7/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 409 kB/s | 222 kB 00:00 2024-08-31T08:17:45.667 INFO:teuthology.orchestra.run.smithi012.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 295 kB/s | 32 kB 00:00 2024-08-31T08:17:45.692 INFO:teuthology.orchestra.run.smithi012.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 140 kB/s | 14 kB 00:00 2024-08-31T08:17:45.717 INFO:teuthology.orchestra.run.smithi012.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-08-31T08:17:45.742 INFO:teuthology.orchestra.run.smithi012.stdout:(11/13): python3-google-auth-2.34.0-1.el9.noarc 1.9 MB/s | 220 kB 00:00 2024-08-31T08:17:45.768 INFO:teuthology.orchestra.run.smithi012.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.7 MB/s | 90 kB 00:00 2024-08-31T08:17:45.801 INFO:teuthology.orchestra.run.smithi012.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 7.7 MB/s | 1.0 MB 00:00 2024-08-31T08:17:45.804 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:45.804 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-08-31T08:17:45.869 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:45.885 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:45.885 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:45.977 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:45.979 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:46.490 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:46.675 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-31T08:17:46.730 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-31T08:17:46.750 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-31T08:17:46.786 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-31T08:17:46.821 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-31T08:17:46.862 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-31T08:17:46.970 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-31T08:17:47.082 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-08-31T08:17:47.132 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-31T08:17:47.792 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-31T08:17:47.820 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 12/12 2024-08-31T08:17:47.908 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-31T08:17:47.960 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-31T08:17:47.994 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-31T08:17:48.043 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-31T08:17:48.073 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 13/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 13/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-31T08:17:48.808 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-31T08:17:48.809 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-31T08:17:49.147 INFO:teuthology.orchestra.run.smithi012.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:49.148 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:49.369 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-mgr-cephadm 2024-08-31T08:17:49.939 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:26 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:50.075 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:50.076 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:50.076 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:50.076 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:50.076 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 154 k 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout:Install 6 Packages 2024-08-31T08:17:50.077 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:50.078 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 7.0 M 2024-08-31T08:17:50.078 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 33 M 2024-08-31T08:17:50.078 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:50.344 INFO:teuthology.orchestra.run.smithi012.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.3 MB/s | 248 kB 00:00 2024-08-31T08:17:50.394 INFO:teuthology.orchestra.run.smithi012.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-08-31T08:17:50.419 INFO:teuthology.orchestra.run.smithi012.stdout:(3/6): ceph-mgr-cephadm-19.3.0-4620.gdc9f1ccb.e 596 kB/s | 154 kB 00:00 2024-08-31T08:17:50.453 INFO:teuthology.orchestra.run.smithi012.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 988 kB/s | 58 kB 00:00 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 12/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1 1/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-08-31T08:17:50.480 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-08-31T08:17:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-08-31T08:17:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-31T08:17:50.481 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-31T08:17:50.495 INFO:teuthology.orchestra.run.smithi012.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 988 kB/s | 74 kB 00:00 2024-08-31T08:17:50.578 INFO:teuthology.orchestra.run.smithi012.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-08-31T08:17:50.581 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:50.581 INFO:teuthology.orchestra.run.smithi012.stdout:Total 14 MB/s | 7.0 MB 00:00 2024-08-31T08:17:50.654 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:50.666 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:50.666 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:50.769 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:50.770 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-diskprediction-local-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-31T08:17:50.873 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-0.3.26-2.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:50.874 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:51.082 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:51.261 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-31T08:17:51.300 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-rook 2024-08-31T08:17:51.323 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-31T08:17:51.878 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:22 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:51.884 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-31T08:17:51.970 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-31T08:17:52.013 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:52.014 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-rook noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 49 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-31T08:17:52.015 INFO:teuthology.orchestra.run.smithi167.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout:Install 13 Packages 2024-08-31T08:17:52.016 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:52.017 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 2.2 M 2024-08-31T08:17:52.017 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 26 M 2024-08-31T08:17:52.017 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:52.054 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-31T08:17:52.084 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 6/6 2024-08-31T08:17:52.264 INFO:teuthology.orchestra.run.smithi167.stdout:(1/13): ceph-mgr-rook-19.3.0-4620.gdc9f1ccb.el9 298 kB/s | 49 kB 00:00 2024-08-31T08:17:52.614 INFO:teuthology.orchestra.run.smithi167.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 37 kB/s | 19 kB 00:00 2024-08-31T08:17:52.648 INFO:teuthology.orchestra.run.smithi167.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 47 kB/s | 26 kB 00:00 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 6/6 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/6 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-31T08:17:52.824 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-31T08:17:53.040 INFO:teuthology.orchestra.run.smithi167.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 286 kB/s | 222 kB 00:00 2024-08-31T08:17:53.202 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:53.203 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:53.382 INFO:teuthology.orchestra.run.smithi167.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 207 kB/s | 159 kB 00:00 2024-08-31T08:17:53.415 INFO:teuthology.orchestra.run.smithi167.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 973 kB/s | 32 kB 00:00 2024-08-31T08:17:53.432 INFO:teuthology.orchestra.run.smithi167.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 850 kB/s | 14 kB 00:00 2024-08-31T08:17:53.444 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-fuse 2024-08-31T08:17:53.458 INFO:teuthology.orchestra.run.smithi167.stdout:(8/13): python3-google-auth-2.34.0-1.el9.noarch 8.6 MB/s | 220 kB 00:00 2024-08-31T08:17:53.500 INFO:teuthology.orchestra.run.smithi167.stdout:(9/13): python3-kubernetes-26.1.0-2.el9.noarch. 25 MB/s | 1.0 MB 00:00 2024-08-31T08:17:53.525 INFO:teuthology.orchestra.run.smithi167.stdout:(10/13): python3-pyasn1-modules-0.4.8-6.el9.noa 318 kB/s | 279 kB 00:00 2024-08-31T08:17:53.550 INFO:teuthology.orchestra.run.smithi167.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2024-08-31T08:17:53.592 INFO:teuthology.orchestra.run.smithi167.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.3 MB/s | 90 kB 00:00 2024-08-31T08:17:53.642 INFO:teuthology.orchestra.run.smithi167.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 89 kB/s | 54 kB 00:00 2024-08-31T08:17:53.646 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:53.646 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-08-31T08:17:53.718 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:17:53.734 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:17:53.734 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:17:53.824 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:17:53.824 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:17:54.004 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:31 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:54.141 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:54.141 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:54.141 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:54.141 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:54.141 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 900 k 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout:Install 2 Packages 2024-08-31T08:17:54.142 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:54.143 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 979 k 2024-08-31T08:17:54.143 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 2.9 M 2024-08-31T08:17:54.143 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:54.349 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:17:54.515 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-31T08:17:54.530 INFO:teuthology.orchestra.run.smithi012.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 616 kB/s | 80 kB 00:00 2024-08-31T08:17:54.577 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-31T08:17:54.641 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-31T08:17:54.678 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-31T08:17:54.720 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-31T08:17:54.747 INFO:teuthology.orchestra.run.smithi012.stdout:(2/2): ceph-fuse-19.3.0-4620.gdc9f1ccb.el9.x86_ 2.5 MB/s | 900 kB 00:00 2024-08-31T08:17:54.747 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:54.748 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.6 MB/s | 979 kB 00:00 2024-08-31T08:17:54.760 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:54.782 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:54.782 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:54.822 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-31T08:17:54.867 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:54.867 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:54.932 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-08-31T08:17:54.997 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-31T08:17:55.061 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:55.221 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-31T08:17:55.289 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/2 2024-08-31T08:17:55.712 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-31T08:17:55.761 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-31T08:17:55.795 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-31T08:17:55.842 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-31T08:17:55.865 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 13/13 2024-08-31T08:17:55.939 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/2 2024-08-31T08:17:55.939 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/2 2024-08-31T08:17:56.192 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-31T08:17:56.192 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:56.192 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:56.193 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-31T08:17:56.193 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:56.193 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:56.492 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install ceph-volume 2024-08-31T08:17:56.662 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 13/13 2024-08-31T08:17:56.662 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-31T08:17:56.663 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-31T08:17:56.975 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-31T08:17:56.975 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:56.975 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:17:56.975 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-rook-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:56.975 INFO:teuthology.orchestra.run.smithi167.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-31T08:17:56.976 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:56.977 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:17:57.063 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:34 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:57.205 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repository Size 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-volume noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 285 k 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 285 k 2024-08-31T08:17:57.206 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 1.3 M 2024-08-31T08:17:57.207 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:57.225 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-mgr-cephadm 2024-08-31T08:17:57.442 INFO:teuthology.orchestra.run.smithi012.stdout:ceph-volume-19.3.0-4620.gdc9f1ccb.el9.noarch.rp 1.2 MB/s | 285 kB 00:00 2024-08-31T08:17:57.442 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:57.443 INFO:teuthology.orchestra.run.smithi012.stdout:Total 1.2 MB/s | 285 kB 00:00 2024-08-31T08:17:57.443 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:57.452 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:57.452 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:17:57.476 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:17:57.477 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:17:57.711 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:17:57.777 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:57.778 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:28 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:17:57.794 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:57.794 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:17:57.794 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-08-31T08:17:57.795 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:57.911 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-cephadm noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 154 k 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-31T08:17:57.912 INFO:teuthology.orchestra.run.smithi167.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-31T08:17:57.913 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:57.913 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:17:57.913 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:17:57.913 INFO:teuthology.orchestra.run.smithi167.stdout:Install 6 Packages 2024-08-31T08:17:57.913 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:17:57.914 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 7.0 M 2024-08-31T08:17:57.914 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 33 M 2024-08-31T08:17:57.914 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:17:58.696 INFO:teuthology.orchestra.run.smithi167.stdout:(1/6): ceph-mgr-cephadm-19.3.0-4620.gdc9f1ccb.e 711 kB/s | 154 kB 00:00 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout: ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:58.736 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:17:58.747 INFO:teuthology.orchestra.run.smithi167.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 11 MB/s | 548 kB 00:00 2024-08-31T08:17:58.772 INFO:teuthology.orchestra.run.smithi167.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 2.3 MB/s | 58 kB 00:00 2024-08-31T08:17:58.789 INFO:teuthology.orchestra.run.smithi167.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 4.3 MB/s | 74 kB 00:00 2024-08-31T08:17:58.981 INFO:teuthology.orchestra.run.smithi167.stdout:(5/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 495 kB/s | 248 kB 00:00 2024-08-31T08:17:59.032 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados-devel 2024-08-31T08:17:59.605 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:36 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:17:59.745 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 126 k 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:17:59.746 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:17:59.747 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 126 k 2024-08-31T08:17:59.747 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 456 k 2024-08-31T08:17:59.747 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:17:59.971 INFO:teuthology.orchestra.run.smithi012.stdout:librados-devel-19.3.0-4620.gdc9f1ccb.el9.x86_64 562 kB/s | 126 kB 00:00 2024-08-31T08:17:59.972 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:17:59.972 INFO:teuthology.orchestra.run.smithi012.stdout:Total 559 kB/s | 126 kB 00:00 2024-08-31T08:17:59.972 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:17:59.978 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:17:59.978 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:18:00.045 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:18:00.045 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:18:00.198 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:18:00.305 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:00.752 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout: librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:01.018 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:01.246 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs2 2024-08-31T08:18:01.815 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:38 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:01.906 INFO:teuthology.orchestra.run.smithi012.stdout:Package libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:01.952 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:01.953 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:01.953 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:01.997 INFO:teuthology.orchestra.run.smithi167.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 1.7 MB/s | 6.0 MB 00:03 2024-08-31T08:18:01.999 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:02.000 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.7 MB/s | 7.0 MB 00:04 2024-08-31T08:18:02.037 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install libcephfs-devel 2024-08-31T08:18:02.073 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:02.085 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:02.085 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:02.184 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:02.185 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:02.503 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:02.597 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:39 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:02.672 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-31T08:18:02.724 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-31T08:18:02.733 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:02.733 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:02.733 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 32 k 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:18:02.734 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:02.735 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 32 k 2024-08-31T08:18:02.735 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 142 k 2024-08-31T08:18:02.735 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:18:02.883 INFO:teuthology.orchestra.run.smithi012.stdout:libcephfs-devel-19.3.0-4620.gdc9f1ccb.el9.x86_6 214 kB/s | 32 kB 00:00 2024-08-31T08:18:02.884 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:02.884 INFO:teuthology.orchestra.run.smithi012.stdout:Total 212 kB/s | 32 kB 00:00 2024-08-31T08:18:02.884 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:18:02.887 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:18:02.888 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:18:02.891 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:18:02.891 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:18:02.976 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:18:03.083 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:03.254 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-31T08:18:03.344 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-31T08:18:03.430 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:03.437 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-31T08:18:03.462 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 6/6 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout: libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:03.649 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:03.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librados2 2024-08-31T08:18:04.225 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 6/6 2024-08-31T08:18:04.286 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/6 2024-08-31T08:18:04.287 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-31T08:18:04.287 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-31T08:18:04.287 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-31T08:18:04.287 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-31T08:18:04.380 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:41 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:04.474 INFO:teuthology.orchestra.run.smithi012.stdout:Package librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-mgr-cephadm-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-31T08:18:04.490 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:04.491 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:04.520 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:04.521 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:04.521 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:04.588 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install librbd1 2024-08-31T08:18:04.746 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-fuse 2024-08-31T08:18:05.157 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:42 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:05.251 INFO:teuthology.orchestra.run.smithi012.stdout:Package librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:05.296 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:05.297 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:05.297 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:05.299 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:36 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:05.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rados 2024-08-31T08:18:05.431 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:05.431 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:05.431 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:18:05.431 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:05.431 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-fuse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 900 k 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout:Installing dependencies: 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout:Install 2 Packages 2024-08-31T08:18:05.432 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:05.433 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 979 k 2024-08-31T08:18:05.433 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 2.9 M 2024-08-31T08:18:05.433 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:05.725 INFO:teuthology.orchestra.run.smithi167.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 443 kB/s | 80 kB 00:00 2024-08-31T08:18:05.817 INFO:teuthology.orchestra.run.smithi167.stdout:(2/2): ceph-fuse-19.3.0-4620.gdc9f1ccb.el9.x86_ 3.2 MB/s | 900 kB 00:00 2024-08-31T08:18:05.817 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:05.818 INFO:teuthology.orchestra.run.smithi167.stdout:Total 2.5 MB/s | 979 kB 00:00 2024-08-31T08:18:05.829 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:05.850 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:05.851 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:05.930 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:05.930 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:05.933 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:42 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:06.025 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:06.071 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:06.072 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:06.072 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:06.130 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:06.137 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rgw 2024-08-31T08:18:06.270 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-31T08:18:06.339 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/2 2024-08-31T08:18:06.704 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:43 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:06.795 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:06.841 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:06.842 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:06.842 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:06.904 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-cephfs 2024-08-31T08:18:06.981 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2/2 2024-08-31T08:18:06.981 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/2 2024-08-31T08:18:07.256 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-31T08:18:07.257 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:07.257 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:07.257 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-31T08:18:07.257 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:07.257 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:07.470 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:44 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:07.543 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install ceph-volume 2024-08-31T08:18:07.561 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:07.607 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:07.608 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:07.608 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:07.667 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install python3-rbd 2024-08-31T08:18:08.112 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:39 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:08.233 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:45 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:08.250 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:08.250 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:08.250 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repository Size 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-volume noarch 2:19.3.0-4620.gdc9f1ccb.el9 ceph-noarch 285 k 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 285 k 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 1.3 M 2024-08-31T08:18:08.251 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:08.325 INFO:teuthology.orchestra.run.smithi012.stdout:Package python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:08.370 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:08.371 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:08.371 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:08.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-fuse 2024-08-31T08:18:08.489 INFO:teuthology.orchestra.run.smithi167.stdout:ceph-volume-19.3.0-4620.gdc9f1ccb.el9.noarch.rp 1.2 MB/s | 285 kB 00:00 2024-08-31T08:18:08.489 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:08.490 INFO:teuthology.orchestra.run.smithi167.stdout:Total 1.2 MB/s | 285 kB 00:00 2024-08-31T08:18:08.490 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:08.499 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:08.499 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:08.522 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:08.522 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:08.738 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:08.799 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:18:08.816 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:18:08.816 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:18:08.816 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2024-08-31T08:18:08.817 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:08.993 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:45 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:09.130 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:09.130 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 85 k 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 85 k 2024-08-31T08:18:09.131 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 231 k 2024-08-31T08:18:09.132 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:18:09.327 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-fuse-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 434 kB/s | 85 kB 00:00 2024-08-31T08:18:09.328 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:09.328 INFO:teuthology.orchestra.run.smithi012.stdout:Total 431 kB/s | 85 kB 00:00 2024-08-31T08:18:09.328 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:18:09.336 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:18:09.336 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:18:09.402 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:18:09.403 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:18:09.547 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:18:09.651 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 1/1 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout: ceph-volume-2:19.3.0-4620.gdc9f1ccb.el9.noarch 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:09.775 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:10.042 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:10.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librados-devel 2024-08-31T08:18:10.290 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:10.291 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:10.291 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:18:10.291 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:10.291 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:10.291 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:10.553 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-mirror 2024-08-31T08:18:10.634 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:41 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:10.771 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout: librados-devel x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 126 k 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:10.772 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:10.773 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:10.773 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 126 k 2024-08-31T08:18:10.773 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 456 k 2024-08-31T08:18:10.773 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:10.978 INFO:teuthology.orchestra.run.smithi167.stdout:librados-devel-19.3.0-4620.gdc9f1ccb.el9.x86_64 616 kB/s | 126 kB 00:00 2024-08-31T08:18:10.979 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:10.979 INFO:teuthology.orchestra.run.smithi167.stdout:Total 611 kB/s | 126 kB 00:00 2024-08-31T08:18:10.979 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:10.985 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:10.985 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:11.050 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:11.050 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:11.123 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:48 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:11.201 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:11.261 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout: Package Arch Version Repo Size 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.2 M 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:18:11.262 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:11.263 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:18:11.263 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:11.263 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 3.2 M 2024-08-31T08:18:11.263 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 14 M 2024-08-31T08:18:11.263 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:18:11.308 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:11.597 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-mirror-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 9.5 MB/s | 3.2 MB 00:00 2024-08-31T08:18:11.599 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:11.599 INFO:teuthology.orchestra.run.smithi012.stdout:Total 9.4 MB/s | 3.2 MB 00:00 2024-08-31T08:18:11.599 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:18:11.608 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:18:11.608 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:18:11.692 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:18:11.692 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:18:11.720 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:11.986 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:11.986 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:11.986 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:11.986 INFO:teuthology.orchestra.run.smithi167.stdout: librados-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:11.987 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:11.987 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:12.148 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:18:12.256 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install libcephfs2 2024-08-31T08:18:12.332 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:12.357 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:12.357 INFO:teuthology.orchestra.run.smithi012.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:18:12.357 INFO:teuthology.orchestra.run.smithi012.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-31T08:18:12.358 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-31T08:18:12.358 INFO:teuthology.orchestra.run.smithi012.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-31T08:18:12.358 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:12.828 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:43 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:12.919 INFO:teuthology.orchestra.run.smithi167.stdout:Package libcephfs2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:12.964 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:12.965 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:12.965 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:13.036 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install libcephfs-devel 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:13.348 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:13.567 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-nbd 2024-08-31T08:18:13.591 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:44 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:13.723 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs-devel x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 32 k 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:13.724 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 32 k 2024-08-31T08:18:13.725 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 142 k 2024-08-31T08:18:13.725 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:13.874 INFO:teuthology.orchestra.run.smithi167.stdout:libcephfs-devel-19.3.0-4620.gdc9f1ccb.el9.x86_6 212 kB/s | 32 kB 00:00 2024-08-31T08:18:13.875 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:13.875 INFO:teuthology.orchestra.run.smithi167.stdout:Total 210 kB/s | 32 kB 00:00 2024-08-31T08:18:13.875 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:13.879 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:13.879 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:13.882 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:13.882 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:13.959 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:14.061 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:14.135 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:51 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:14.273 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:14.273 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 170 k 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Install 1 Package 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 170 k 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 490 k 2024-08-31T08:18:14.274 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-08-31T08:18:14.413 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:14.484 INFO:teuthology.orchestra.run.smithi012.stdout:rbd-nbd-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 813 kB/s | 170 kB 00:00 2024-08-31T08:18:14.485 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:14.485 INFO:teuthology.orchestra.run.smithi012.stdout:Total 808 kB/s | 170 kB 00:00 2024-08-31T08:18:14.485 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-08-31T08:18:14.493 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-08-31T08:18:14.493 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-08-31T08:18:14.560 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-08-31T08:18:14.560 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout: libcephfs-devel-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:14.638 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:14.724 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-08-31T08:18:14.825 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:14.839 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librados2 2024-08-31T08:18:15.216 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:15.391 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:46 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout: rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:15.456 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:15.481 INFO:teuthology.orchestra.run.smithi167.stdout:Package librados2-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:15.525 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:15.526 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:15.526 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:15.596 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install librbd1 2024-08-31T08:18:15.677 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum -y install rbd-mirror 2024-08-31T08:18:16.149 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:47 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:16.239 INFO:teuthology.orchestra.run.smithi167.stdout:Package librbd1-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:16.245 INFO:teuthology.orchestra.run.smithi012.stdout:Last metadata expiration check: 0:01:53 ago on Sat 31 Aug 2024 08:16:23 AM UTC. 2024-08-31T08:18:16.283 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:16.284 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:16.284 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:16.339 INFO:teuthology.orchestra.run.smithi012.stdout:Package rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:16.354 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rados 2024-08-31T08:18:16.384 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-08-31T08:18:16.385 INFO:teuthology.orchestra.run.smithi012.stdout:Nothing to do. 2024-08-31T08:18:16.385 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-08-31T08:18:16.469 DEBUG:teuthology.parallel:result is None 2024-08-31T08:18:16.910 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:47 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:16.999 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rados-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:17.043 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:17.044 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:17.044 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:17.113 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rgw 2024-08-31T08:18:17.668 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:48 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:17.757 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rgw-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:17.801 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:17.802 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:17.802 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:17.870 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-cephfs 2024-08-31T08:18:18.460 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:49 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:18.554 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-cephfs-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:18.599 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:18.600 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:18.600 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:18.676 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install python3-rbd 2024-08-31T08:18:19.262 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:50 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:19.351 INFO:teuthology.orchestra.run.smithi167.stdout:Package python3-rbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:19.394 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:19.395 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:19.395 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:19.471 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-fuse 2024-08-31T08:18:20.023 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:51 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:20.155 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:20.156 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:20.156 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2024-08-31T08:18:20.156 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:20.156 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-fuse x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 85 k 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 85 k 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 231 k 2024-08-31T08:18:20.157 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:20.354 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-fuse-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 433 kB/s | 85 kB 00:00 2024-08-31T08:18:20.354 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:20.354 INFO:teuthology.orchestra.run.smithi167.stdout:Total 430 kB/s | 85 kB 00:00 2024-08-31T08:18:20.354 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:20.361 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:20.361 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:20.426 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:20.426 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:20.571 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:20.682 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:21.057 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:21.301 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:21.301 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:21.301 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:21.302 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-fuse-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:21.302 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:21.302 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:21.565 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-mirror 2024-08-31T08:18:22.135 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:53 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:22.270 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout: Package Arch Version Repo Size 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-mirror x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 3.2 M 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:22.271 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:22.272 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:22.272 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:22.272 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 3.2 M 2024-08-31T08:18:22.272 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 14 M 2024-08-31T08:18:22.272 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:22.599 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-mirror-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 9.7 MB/s | 3.2 MB 00:00 2024-08-31T08:18:22.600 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:22.600 INFO:teuthology.orchestra.run.smithi167.stdout:Total 9.7 MB/s | 3.2 MB 00:00 2024-08-31T08:18:22.601 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:22.609 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:22.609 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:22.690 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:22.690 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:23.134 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:23.300 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:23.326 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:23.326 INFO:teuthology.orchestra.run.smithi167.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-31T08:18:23.327 INFO:teuthology.orchestra.run.smithi167.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-31T08:18:23.327 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-31T08:18:23.327 INFO:teuthology.orchestra.run.smithi167.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-31T08:18:23.327 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:24.297 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:24.561 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-nbd 2024-08-31T08:18:25.122 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:56 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:25.264 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout: Package Architecture Version Repository Size 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout:Installing: 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-nbd x86_64 2:19.3.0-4620.gdc9f1ccb.el9 ceph 170 k 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:25.265 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction Summary 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout:================================================================================ 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout:Install 1 Package 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout:Total download size: 170 k 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout:Installed size: 490 k 2024-08-31T08:18:25.266 INFO:teuthology.orchestra.run.smithi167.stdout:Downloading Packages: 2024-08-31T08:18:25.478 INFO:teuthology.orchestra.run.smithi167.stdout:rbd-nbd-19.3.0-4620.gdc9f1ccb.el9.x86_64.rpm 803 kB/s | 170 kB 00:00 2024-08-31T08:18:25.479 INFO:teuthology.orchestra.run.smithi167.stdout:-------------------------------------------------------------------------------- 2024-08-31T08:18:25.479 INFO:teuthology.orchestra.run.smithi167.stdout:Total 797 kB/s | 170 kB 00:00 2024-08-31T08:18:25.479 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction check 2024-08-31T08:18:25.487 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction check succeeded. 2024-08-31T08:18:25.487 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction test 2024-08-31T08:18:25.552 INFO:teuthology.orchestra.run.smithi167.stdout:Transaction test succeeded. 2024-08-31T08:18:25.552 INFO:teuthology.orchestra.run.smithi167.stdout:Running transaction 2024-08-31T08:18:25.704 INFO:teuthology.orchestra.run.smithi167.stdout: Preparing : 1/1 2024-08-31T08:18:25.800 INFO:teuthology.orchestra.run.smithi167.stdout: Installing : rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:26.227 INFO:teuthology.orchestra.run.smithi167.stdout: Running scriptlet: rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:26.504 INFO:teuthology.orchestra.run.smithi167.stdout: Verifying : rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 1/1 2024-08-31T08:18:26.504 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:26.504 INFO:teuthology.orchestra.run.smithi167.stdout:Installed: 2024-08-31T08:18:26.504 INFO:teuthology.orchestra.run.smithi167.stdout: rbd-nbd-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 2024-08-31T08:18:26.505 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:26.505 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:26.762 DEBUG:teuthology.orchestra.run.smithi167:> sudo yum -y install rbd-mirror 2024-08-31T08:18:27.333 INFO:teuthology.orchestra.run.smithi167.stdout:Last metadata expiration check: 0:01:58 ago on Sat 31 Aug 2024 08:16:29 AM UTC. 2024-08-31T08:18:27.425 INFO:teuthology.orchestra.run.smithi167.stdout:Package rbd-mirror-2:19.3.0-4620.gdc9f1ccb.el9.x86_64 is already installed. 2024-08-31T08:18:27.470 INFO:teuthology.orchestra.run.smithi167.stdout:Dependencies resolved. 2024-08-31T08:18:27.470 INFO:teuthology.orchestra.run.smithi167.stdout:Nothing to do. 2024-08-31T08:18:27.471 INFO:teuthology.orchestra.run.smithi167.stdout:Complete! 2024-08-31T08:18:27.548 DEBUG:teuthology.parallel:result is None 2024-08-31T08:18:27.549 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:18:27.703 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-31T08:18:27.729 INFO:teuthology.orchestra.run.smithi012.stdout:19.3.0-4620.gdc9f1ccb.el9 2024-08-31T08:18:27.730 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4620.gdc9f1ccb.el9 2024-08-31T08:18:27.730 INFO:teuthology.task.install:The correct ceph version 19.3.0-4620.gdc9f1ccb is installed. 2024-08-31T08:18:27.732 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:18:27.885 DEBUG:teuthology.orchestra.run.smithi167:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-31T08:18:27.906 INFO:teuthology.orchestra.run.smithi167.stdout:19.3.0-4620.gdc9f1ccb.el9 2024-08-31T08:18:27.907 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-4620.gdc9f1ccb.el9 2024-08-31T08:18:27.907 INFO:teuthology.task.install:The correct ceph version 19.3.0-4620.gdc9f1ccb is installed. 2024-08-31T08:18:27.909 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-31T08:18:27.909 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:27.909 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-31T08:18:27.939 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:27.939 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-31T08:18:27.973 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-31T08:18:27.973 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:27.973 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/daemon-helper 2024-08-31T08:18:28.008 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-31T08:18:28.076 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:28.077 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/daemon-helper 2024-08-31T08:18:28.101 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-31T08:18:28.166 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-31T08:18:28.166 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:28.167 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-31T08:18:28.193 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-31T08:18:28.260 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:28.261 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-31T08:18:28.285 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-31T08:18:28.350 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-31T08:18:28.350 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:28.350 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/usr/bin/stdin-killer 2024-08-31T08:18:28.377 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-31T08:18:28.444 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:28.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/usr/bin/stdin-killer 2024-08-31T08:18:28.469 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-31T08:18:28.533 INFO:teuthology.run_tasks:Running task ceph... 2024-08-31T08:18:28.606 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-08-31T08:18:28.606 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 777 /var/log/ceph 2024-08-31T08:18:28.609 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 777 /var/log/ceph 2024-08-31T08:18:28.634 INFO:tasks.ceph:Disabling ceph logrotate... 2024-08-31T08:18:28.634 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /etc/logrotate.d/ceph 2024-08-31T08:18:28.676 DEBUG:teuthology.orchestra.run.smithi167:> sudo rm -f -- /etc/logrotate.d/ceph 2024-08-31T08:18:28.701 INFO:tasks.ceph:Creating extra log directories... 2024-08-31T08:18:28.701 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-08-31T08:18:28.743 DEBUG:teuthology.orchestra.run.smithi167:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-08-31T08:18:28.779 INFO:tasks.ceph:Creating ceph cluster cluster1... 2024-08-31T08:18:28.779 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-08-31T08:18:28.780 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi/7883237', 'branch': 'wip-dis-testing', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}', 'email': 'idryomov@gmail.com', 'first_in_suite': False, 'job_id': '7883237', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-dis-testing'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-dis-testing', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, 'owner': 'scheduled_dis@teuthology', 'priority': 75, 'repo': 'https://github.com/ceph/ceph-ci', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 4871, 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'sleep_before_teardown': 0, 'subset': '98/100', 'suite': 'rbd', 'suite_branch': 'wip-dis-testing', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci', 'suite_sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'targets': {'smithi012.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNEx9XsXup32GoJWsUuPxH1n47mKHEI6yaBmm41mtD7h8aNmNH7iD1ARqK8OBx22xCY1NFPDkU2kwMBJEqjSn7E=', 'smithi167.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpbZp5xuE44w4oUgvkpwPbPMuNfddfjlOsUe5biVwgLuxRVcedQHT4m1bf7uePACsma12cDwmAwkJhAgE8y+p8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-08-31_07:23:49', 'tube': 'smithi', 'user': 'dis', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-08-31T08:18:28.780 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-08-31T08:18:28.810 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2024-08-31T08:18:28.826 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m0777 -- /var/run/ceph 2024-08-31T08:18:28.867 DEBUG:teuthology.orchestra.run.smithi167:> sudo install -d -m0777 -- /var/run/ceph 2024-08-31T08:18:28.891 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:28.891 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/scratch_devs of=/dev/stdout 2024-08-31T08:18:28.947 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-31T08:18:28.948 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_1 2024-08-31T08:18:29.003 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-31T08:18:29.003 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:18:29.003 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-08-31 08:18:12.632618268 +0000 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-08-31 08:15:45.162663101 +0000 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-08-31 08:15:45.162663101 +0000 2024-08-31T08:18:29.004 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-08-31 08:15:45.162663101 +0000 2024-08-31T08:18:29.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-31T08:18:29.068 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-08-31T08:18:29.068 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-08-31T08:18:29.068 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.00020894 s, 2.5 MB/s 2024-08-31T08:18:29.070 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-31T08:18:29.127 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_2 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-08-31 08:18:12.632618268 +0000 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-08-31 08:15:45.506663144 +0000 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-08-31 08:15:45.506663144 +0000 2024-08-31T08:18:29.184 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-08-31 08:15:45.506663144 +0000 2024-08-31T08:18:29.184 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-31T08:18:29.248 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-08-31T08:18:29.248 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-08-31T08:18:29.249 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000184805 s, 2.8 MB/s 2024-08-31T08:18:29.250 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-31T08:18:29.306 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_3 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:18:29.362 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-08-31 08:18:12.633618258 +0000 2024-08-31T08:18:29.363 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-08-31 08:15:45.838663186 +0000 2024-08-31T08:18:29.363 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-08-31 08:15:45.838663186 +0000 2024-08-31T08:18:29.363 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-08-31 08:15:45.838663186 +0000 2024-08-31T08:18:29.363 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-31T08:18:29.426 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-08-31T08:18:29.426 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-08-31T08:18:29.426 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000173778 s, 2.9 MB/s 2024-08-31T08:18:29.428 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-31T08:18:29.484 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_4 2024-08-31T08:18:29.540 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-08-31 08:18:12.633618258 +0000 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-08-31 08:15:46.171663228 +0000 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-08-31 08:15:46.171663228 +0000 2024-08-31T08:18:29.541 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-08-31 08:15:46.171663228 +0000 2024-08-31T08:18:29.541 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-31T08:18:29.603 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-08-31T08:18:29.604 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-08-31T08:18:29.604 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000191604 s, 2.7 MB/s 2024-08-31T08:18:29.605 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-31T08:18:29.661 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:18:29.661 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-08-31T08:18:29.661 INFO:tasks.ceph:Generating config... 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client] debug rbd = 20 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client] log to stderr = False 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client] rbd default features = 125 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:29.663 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:29.664 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:29.664 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:29.664 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-08-31T08:18:29.664 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-08-31T08:18:29.664 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-08-31T08:18:29.664 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-08-31T08:18:29.664 INFO:tasks.ceph:[mgr] debug ms = 1 2024-08-31T08:18:29.664 INFO:tasks.ceph:[mon] debug mon = 20 2024-08-31T08:18:29.664 INFO:tasks.ceph:[mon] debug ms = 1 2024-08-31T08:18:29.664 INFO:tasks.ceph:[mon] debug paxos = 20 2024-08-31T08:18:29.664 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2024-08-31T08:18:29.664 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2024-08-31T08:18:29.664 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] debug ms = 1 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] debug osd = 20 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-08-31T08:18:29.665 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-08-31T08:18:29.665 INFO:tasks.ceph:Setting up cluster1.mon.a... 2024-08-31T08:18:29.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2024-08-31T08:18:29.749 INFO:teuthology.orchestra.run.smithi012.stdout:creating /etc/ceph/cluster1.keyring 2024-08-31T08:18:29.752 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2024-08-31T08:18:29.793 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-08-31T08:18:29.857 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '172.21.15.12')] 2024-08-31T08:18:29.858 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.12', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-08-31T08:18:29.859 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:29.859 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-08-31T08:18:29.912 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.12 --print /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:29.992 INFO:teuthology.orchestra.run.smithi012.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:29.992 INFO:teuthology.orchestra.run.smithi012.stdout:monmaptool: generated fsid c26ce5cc-1226-40b9-9482-3cb99b32adb2 2024-08-31T08:18:29.992 INFO:teuthology.orchestra.run.smithi012.stdout:setting min_mon_release = quincy 2024-08-31T08:18:29.992 INFO:teuthology.orchestra.run.smithi012.stdout:epoch 0 2024-08-31T08:18:29.992 INFO:teuthology.orchestra.run.smithi012.stdout:fsid c26ce5cc-1226-40b9-9482-3cb99b32adb2 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:last_changed 2024-08-31T08:18:29.992362+0000 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:created 2024-08-31T08:18:29.992362+0000 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:min_mon_release 17 (quincy) 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:election_strategy: 1 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.a 2024-08-31T08:18:29.993 INFO:teuthology.orchestra.run.smithi012.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2024-08-31T08:18:29.994 DEBUG:teuthology.orchestra.run.smithi012:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-08-31T08:18:30.048 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID c26ce5cc-1226-40b9-9482-3cb99b32adb2... 2024-08-31T08:18:30.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-08-31T08:18:30.091 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout:[global] 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: chdir = "" 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: auth supported = cephx 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: filestore xattr use omap = true 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: mon clock drift allowed = 1.000 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: osd crush chooseleaf type = 0 2024-08-31T08:18:30.131 INFO:teuthology.orchestra.run.smithi012.stdout: auth debug = true 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: ms die on old message = true 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: ms die on bug = true 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: mon pg warn max object skew = 0 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: # disable pg_autoscaler by default for new pools 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: osd pool default size = 2 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd allow primary affinity = true 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd allow pg remap = true 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on legacy crush tunables = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on crush straw calc version zero = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on no sortbitwise = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on osd down out interval zero = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on too few osds = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: mon_allow_pool_size_one = true 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.133 INFO:teuthology.orchestra.run.smithi012.stdout: osd default data pool replay window = 5 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon allow pool delete = true 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon cluster log file level = debug 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: debug asserts on shutdown = true 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon health detail to clog = false 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon host = 172.21.15.12 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon client directed command retry = 5 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on pool no app = False 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: ms inject socket failures = 5000 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: fsid = c26ce5cc-1226-40b9-9482-3cb99b32adb2 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.134 INFO:teuthology.orchestra.run.smithi012.stdout:[osd] 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd journal size = 100 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd scrub load threshold = 5.0 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd scrub max interval = 600 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd mclock profile = high_recovery_ops 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd recover clone overlap = true 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd recovery max chunk = 1048576 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug shutdown = true 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug op order = true 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug verify stray on activate = true 2024-08-31T08:18:30.135 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug trim objects = true 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: osd open classes on start = true 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug pg log writeout = true 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: osd deep scrub update digest min age = 30 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: osd map max advance = 10 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: journal zero on create = true 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: filestore ondisk finisher threads = 3 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: filestore apply finisher threads = 3 2024-08-31T08:18:30.136 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: bdev debug aio = true 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug misdirected ops = true 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: bluefs allocator = hybrid 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore allocator = hybrid 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore block size = 96636764160 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore fsck on mount = True 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: debug bluefs = 1/20 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: debug bluestore = 1/20 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: debug osd = 20 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: debug rocksdb = 4/10 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd backfillfull_ratio = 0.85 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd full ratio = 0.9 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd nearfull ratio = 0.8 2024-08-31T08:18:30.137 INFO:teuthology.orchestra.run.smithi012.stdout: osd failsafe full ratio = 0.95 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: osd objectstore = bluestore 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout:[mgr] 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug mgr = 20 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug mon = 20 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug auth = 20 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: mgr/telemetry/nag = false 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout:[mon] 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug mon = 20 2024-08-31T08:18:30.138 INFO:teuthology.orchestra.run.smithi012.stdout: debug paxos = 20 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: debug auth = 20 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon data avail warn = 5 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon mgr mkfs grace = 240 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd reporter subtree level = osd 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd prime pg temp = true 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: auth mon ticket ttl = 660 # 11m 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: auth service ticket ttl = 240 # 4m 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: # don't complain about insecure global_id in the test suite 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-31T08:18:30.139 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: # 1m isn't quite enough 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: mon_down_mkfs_grace = 2m 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_filestore_osds = false 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout:[client] 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: rgw cache enabled = true 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: rgw enable ops log = true 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: rgw enable usage log = true 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: debug rbd = 20 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: debug rbd_mirror = 15 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: log to stderr = False 2024-08-31T08:18:30.140 INFO:teuthology.orchestra.run.smithi012.stdout: rbd default features = 125 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: rbd mirror image policy type = none 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout:[mon.a] 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.0] 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.1] 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.2] 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.3] 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.141 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout:[global] 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout: chdir = "" 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout: auth supported = cephx 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.142 INFO:teuthology.orchestra.run.smithi167.stdout: filestore xattr use omap = true 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: mon clock drift allowed = 1.000 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: osd crush chooseleaf type = 0 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: auth debug = true 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: ms die on old message = true 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: ms die on bug = true 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: mon pg warn max object skew = 0 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.143 INFO:teuthology.orchestra.run.smithi167.stdout: # disable pg_autoscaler by default for new pools 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: osd pool default size = 2 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd allow primary affinity = true 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd allow pg remap = true 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on legacy crush tunables = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on crush straw calc version zero = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on no sortbitwise = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on osd down out interval zero = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on too few osds = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-31T08:18:30.144 INFO:teuthology.orchestra.run.smithi167.stdout: mon_allow_pool_size_one = true 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: osd default data pool replay window = 5 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon allow pool delete = true 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon cluster log file level = debug 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: debug asserts on shutdown = true 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon health detail to clog = false 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon host = 172.21.15.12 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon client directed command retry = 5 2024-08-31T08:18:30.145 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on pool no app = False 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: ms inject socket failures = 5000 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: fsid = c26ce5cc-1226-40b9-9482-3cb99b32adb2 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout:[osd] 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd journal size = 100 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd scrub load threshold = 5.0 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd scrub max interval = 600 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd mclock profile = high_recovery_ops 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd recover clone overlap = true 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: osd recovery max chunk = 1048576 2024-08-31T08:18:30.146 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug shutdown = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug op order = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug verify stray on activate = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug trim objects = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd open classes on start = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug pg log writeout = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd deep scrub update digest min age = 30 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: osd map max advance = 10 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: journal zero on create = true 2024-08-31T08:18:30.147 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: filestore ondisk finisher threads = 3 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: filestore apply finisher threads = 3 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: bdev debug aio = true 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug misdirected ops = true 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: bluefs allocator = hybrid 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore allocator = hybrid 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore block size = 96636764160 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore fsck on mount = True 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: debug bluefs = 1/20 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: debug bluestore = 1/20 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: debug osd = 20 2024-08-31T08:18:30.148 INFO:teuthology.orchestra.run.smithi167.stdout: debug rocksdb = 4/10 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd backfillfull_ratio = 0.85 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd full ratio = 0.9 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd nearfull ratio = 0.8 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: osd failsafe full ratio = 0.95 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: osd objectstore = bluestore 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout:[mgr] 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: debug mgr = 20 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: debug mon = 20 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: debug auth = 20 2024-08-31T08:18:30.149 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mgr/telemetry/nag = false 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout:[mon] 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: debug mon = 20 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: debug paxos = 20 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: debug auth = 20 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mon data avail warn = 5 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mon mgr mkfs grace = 240 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:18:30.150 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd reporter subtree level = osd 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd prime pg temp = true 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: auth mon ticket ttl = 660 # 11m 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: auth service ticket ttl = 240 # 4m 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: # don't complain about insecure global_id in the test suite 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: # 1m isn't quite enough 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: mon_down_mkfs_grace = 2m 2024-08-31T08:18:30.151 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_filestore_osds = false 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout:[client] 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: rgw cache enabled = true 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: rgw enable ops log = true 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: rgw enable usage log = true 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: debug rbd = 20 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: debug rbd_mirror = 15 2024-08-31T08:18:30.152 INFO:teuthology.orchestra.run.smithi167.stdout: log to stderr = False 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: rbd default features = 125 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: rbd mirror image policy type = none 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout:[mon.a] 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.0] 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.1] 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.2] 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.153 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.3] 2024-08-31T08:18:30.154 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:18:30.154 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:18:30.155 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2024-08-31T08:18:30.155 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2024-08-31T08:18:30.249 INFO:tasks.ceph:Copying monmap to all nodes... 2024-08-31T08:18:30.249 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:30.249 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2024-08-31T08:18:30.264 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:30.265 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2024-08-31T08:18:30.319 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:18:30.320 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:30.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/cluster1.keyring 2024-08-31T08:18:30.320 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-08-31T08:18:30.393 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:30.393 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:30.449 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:18:30.449 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:30.449 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/cluster1.keyring 2024-08-31T08:18:30.449 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2024-08-31T08:18:30.491 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:30.491 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:30.548 INFO:tasks.ceph:Setting up mon nodes... 2024-08-31T08:18:30.549 INFO:tasks.ceph:Setting up mgr nodes... 2024-08-31T08:18:30.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2024-08-31T08:18:30.599 INFO:teuthology.orchestra.run.smithi012.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2024-08-31T08:18:30.601 INFO:tasks.ceph:Setting up mds nodes... 2024-08-31T08:18:30.602 INFO:tasks.ceph_client:Setting up client nodes... 2024-08-31T08:18:30.602 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2024-08-31T08:18:30.642 INFO:teuthology.orchestra.run.smithi012.stdout:creating /etc/ceph/cluster1.client.0.keyring 2024-08-31T08:18:30.657 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2024-08-31T08:18:30.721 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2024-08-31T08:18:30.734 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2024-08-31T08:18:30.776 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2024-08-31T08:18:30.789 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2024-08-31T08:18:30.870 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2024-08-31T08:18:30.882 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2024-08-31T08:18:30.964 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2024-08-31T08:18:30.977 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2024-08-31T08:18:31.058 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2024-08-31T08:18:31.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2024-08-31T08:18:31.153 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2024-08-31T08:18:31.166 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2024-08-31T08:18:31.247 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2024-08-31T08:18:31.260 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2024-08-31T08:18:31.340 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2024-08-31T08:18:31.353 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-08-31T08:18:31.353 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2024-08-31T08:18:31.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2024-08-31T08:18:31.381 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:18:31.381 INFO:tasks.ceph:role: cluster1.osd.0 2024-08-31T08:18:31.381 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:18:31.381 DEBUG:teuthology.orchestra.run.smithi012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:18:31.457 INFO:teuthology.orchestra.run.smithi012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:18:31.963 INFO:teuthology.orchestra.run.smithi012.stdout:Discarding blocks...Done. 2024-08-31T08:18:31.965 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi012.front.sepia.ceph.com -o noatime 2024-08-31T08:18:31.965 DEBUG:teuthology.orchestra.run.smithi012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2024-08-31T08:18:32.107 DEBUG:teuthology.orchestra.run.smithi012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2024-08-31T08:18:32.142 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2024-08-31T08:18:32.210 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:18:32.210 INFO:tasks.ceph:role: cluster1.osd.1 2024-08-31T08:18:32.210 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:18:32.210 DEBUG:teuthology.orchestra.run.smithi012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:18:32.285 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:18:32.286 INFO:teuthology.orchestra.run.smithi012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:18:32.728 INFO:teuthology.orchestra.run.smithi012.stdout:Discarding blocks...Done. 2024-08-31T08:18:32.731 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi012.front.sepia.ceph.com -o noatime 2024-08-31T08:18:32.731 DEBUG:teuthology.orchestra.run.smithi012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2024-08-31T08:18:32.824 DEBUG:teuthology.orchestra.run.smithi012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2024-08-31T08:18:32.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2024-08-31T08:18:32.917 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:18:32.917 INFO:tasks.ceph:role: cluster1.osd.2 2024-08-31T08:18:32.918 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:18:32.918 DEBUG:teuthology.orchestra.run.smithi012:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-08-31T08:18:32.990 INFO:teuthology.orchestra.run.smithi012.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:18:32.990 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:18:32.991 INFO:teuthology.orchestra.run.smithi012.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:18:33.492 INFO:teuthology.orchestra.run.smithi012.stdout:Discarding blocks...Done. 2024-08-31T08:18:33.494 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi012.front.sepia.ceph.com -o noatime 2024-08-31T08:18:33.495 DEBUG:teuthology.orchestra.run.smithi012:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2024-08-31T08:18:33.639 DEBUG:teuthology.orchestra.run.smithi012:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2024-08-31T08:18:33.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:33.760 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:33.760+0000 7f6184015640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2024-08-31T08:18:33.766 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:33.765+0000 7f6184015640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2024-08-31T08:18:33.766 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:33.765+0000 7f6184015640 -1 bdev(0x55c24024b800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2024-08-31T08:18:33.766 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:33.765+0000 7f6184015640 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2024-08-31T08:18:36.918 DEBUG:teuthology.orchestra.run.smithi012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2024-08-31T08:18:36.948 DEBUG:teuthology.orchestra.run.smithi012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:37.038 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:37.037+0000 7f2f25022640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2024-08-31T08:18:37.038 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:37.037+0000 7f2f25022640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2024-08-31T08:18:37.038 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:37.037+0000 7f2f25022640 -1 bdev(0x5587b29a7800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2024-08-31T08:18:37.038 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:37.037+0000 7f2f25022640 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2024-08-31T08:18:39.947 DEBUG:teuthology.orchestra.run.smithi012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2024-08-31T08:18:39.976 DEBUG:teuthology.orchestra.run.smithi012:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:40.067 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:40.067+0000 7f24d713e640 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2024-08-31T08:18:40.068 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:40.067+0000 7f24d713e640 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2024-08-31T08:18:40.068 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:40.067+0000 7f24d713e640 -1 bdev(0x5555f1c37800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2024-08-31T08:18:40.068 INFO:teuthology.orchestra.run.smithi012.stderr:2024-08-31T08:18:40.067+0000 7f24d713e640 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2024-08-31T08:18:42.984 DEBUG:teuthology.orchestra.run.smithi012:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2024-08-31T08:18:43.011 INFO:tasks.ceph:Reading keys from all nodes... 2024-08-31T08:18:43.012 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:43.012 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2024-08-31T08:18:43.078 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:43.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2024-08-31T08:18:43.145 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:43.145 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2024-08-31T08:18:43.212 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:43.212 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2024-08-31T08:18:43.277 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:43.278 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2024-08-31T08:18:43.334 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.334 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2024-08-31T08:18:43.353 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.353 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2024-08-31T08:18:43.411 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.411 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2024-08-31T08:18:43.468 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.468 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2024-08-31T08:18:43.526 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.526 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2024-08-31T08:18:43.585 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.585 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2024-08-31T08:18:43.641 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.642 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2024-08-31T08:18:43.699 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:18:43.699 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2024-08-31T08:18:43.755 INFO:tasks.ceph:Adding keys to all mons... 2024-08-31T08:18:43.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo tee -a /etc/ceph/cluster1.keyring 2024-08-31T08:18:43.780 INFO:teuthology.orchestra.run.smithi012.stdout:[mgr.x] 2024-08-31T08:18:43.780 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJmDqO6IxAAaUx6dZ0QWiUPy9dfV/ZZuw== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[osd.0] 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDZ0dJmU0laLRAAT6aAsGJWSnn9W7YApc3STg== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[osd.1] 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDd0dJmsyhCAhAAL3UcBIvdRt489lracSS2Gw== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[osd.2] 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDg0dJm3WQKBBAAibDtVssYbpWfA4MrHWHLIQ== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[client.0] 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJm2zxRJhAAItUY5SPgLglEFmBbEoCo0w== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror] 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJmrw/6KhAACJu5QZpXVnZm0wv0EgS4Dw== 2024-08-31T08:18:43.781 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.0] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJm4QlLLhAANebu3lwKrCTMMpbQVurUcg== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.1] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJmGLDhMxAAB7bY6pr+8sc1+iXsi2f3dQ== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.2] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDW0dJmL7p5ORAAEZkhhom3ksV2CIBBK01zng== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.3] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDX0dJmIbp8AxAA+2aYROwihhlV773Ig8N1Mg== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.4] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDX0dJmTJ8jCRAANy68JRCisSKomLlCAS9GIg== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.5] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDX0dJmX2a9DhAAMMGrzbmi/U3k3Ul8f6LUCw== 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.6] 2024-08-31T08:18:43.782 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQDX0dJmibBOFBAAw4H4MEhi/x8gGOeBfQvaqQ== 2024-08-31T08:18:43.783 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-31T08:18:43.877 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:18:43.928 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:18:43.979 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:18:44.029 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.083 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.189 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.296 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.353 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.406 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.460 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:18:44.513 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-08-31T08:18:44.513 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2024-08-31T08:18:44.537 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2024-08-31T08:18:44.632 DEBUG:teuthology.orchestra.run.smithi012:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2024-08-31T08:18:44.656 DEBUG:teuthology.orchestra.run.smithi012:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2024-08-31T08:18:44.711 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2024-08-31T08:18:44.712 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2024-08-31T08:18:44.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2024-08-31T08:18:44.754 INFO:tasks.ceph.cluster1.mon.a:Started 2024-08-31T08:18:44.755 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2024-08-31T08:18:44.755 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2024-08-31T08:18:44.755 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2024-08-31T08:18:44.758 INFO:tasks.ceph.cluster1.mgr.x:Started 2024-08-31T08:18:44.758 DEBUG:tasks.ceph:set 0 configs 2024-08-31T08:18:44.758 INFO:tasks.ceph:Setting crush tunables to default 2024-08-31T08:18:44.758 DEBUG:teuthology.orchestra.run.smithi012:> sudo ceph --cluster cluster1 osd crush tunables default 2024-08-31T08:18:44.933 INFO:teuthology.orchestra.run.smithi012.stderr:adjusted tunables profile to default 2024-08-31T08:18:44.944 INFO:tasks.ceph:check_enable_crimson: False 2024-08-31T08:18:44.944 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2024-08-31T08:18:44.944 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:44.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2024-08-31T08:18:44.968 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:44.968 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2024-08-31T08:18:45.034 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:18:45.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2024-08-31T08:18:45.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo ceph --cluster cluster1 osd new e6db31da-6057-4ef4-83ac-325ca3fc14c7 0 2024-08-31T08:18:45.320 INFO:teuthology.orchestra.run.smithi012.stdout:0 2024-08-31T08:18:45.331 DEBUG:teuthology.orchestra.run.smithi012:> sudo ceph --cluster cluster1 osd new 425794f8-f55e-44b1-a291-a3d71b3b00d6 1 2024-08-31T08:18:45.510 INFO:teuthology.orchestra.run.smithi012.stdout:1 2024-08-31T08:18:45.521 DEBUG:teuthology.orchestra.run.smithi012:> sudo ceph --cluster cluster1 osd new 09b2fb83-7065-4d81-9966-bee8cdc8886c 2 2024-08-31T08:18:45.701 INFO:teuthology.orchestra.run.smithi012.stdout:2 2024-08-31T08:18:45.712 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2024-08-31T08:18:45.712 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2024-08-31T08:18:45.714 INFO:tasks.ceph.cluster1.osd.0:Started 2024-08-31T08:18:45.715 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2024-08-31T08:18:45.715 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2024-08-31T08:18:45.716 INFO:tasks.ceph.cluster1.osd.1:Started 2024-08-31T08:18:45.717 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2024-08-31T08:18:45.717 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2024-08-31T08:18:45.719 INFO:tasks.ceph.cluster1.osd.2:Started 2024-08-31T08:18:45.719 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:18:45.912 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:18:45.913 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":5,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:18:45.700368+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:18:45.923 INFO:tasks.ceph.ceph_manager.cluster1:[] 2024-08-31T08:18:45.923 INFO:tasks.ceph:Waiting for OSDs to come up 2024-08-31T08:18:46.225 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:18:46.387 INFO:teuthology.misc.health.smithi012.stdout: 2024-08-31T08:18:46.387 INFO:teuthology.misc.health.smithi012.stdout:{"epoch":5,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:18:45.700368+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:18:46.397 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-08-31T08:18:47.176 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:18:47.175+0000 7f6c9a21c640 -1 Falling back to public interface 2024-08-31T08:18:47.181 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:18:47.180+0000 7f9dd45d4640 -1 Falling back to public interface 2024-08-31T08:18:47.185 INFO:tasks.ceph.cluster1.osd.1.smithi012.stderr:2024-08-31T08:18:47.184+0000 7fcaa60e2640 -1 Falling back to public interface 2024-08-31T08:18:50.317 INFO:tasks.ceph.cluster1.mgr.x.smithi012.stderr:2024-08-31T08:18:50.316+0000 7ff93ddb4640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-31T08:18:51.032 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:18:51.031+0000 7f6c9a21c640 -1 osd.0 0 log_to_monitors true 2024-08-31T08:18:51.270 INFO:tasks.ceph.cluster1.osd.1.smithi012.stderr:2024-08-31T08:18:51.269+0000 7fcaa60e2640 -1 osd.1 0 log_to_monitors true 2024-08-31T08:18:51.274 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:18:51.273+0000 7f9dd45d4640 -1 osd.2 0 log_to_monitors true 2024-08-31T08:18:52.701 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:18:53.021 INFO:teuthology.misc.health.smithi012.stdout: 2024-08-31T08:18:53.022 INFO:teuthology.misc.health.smithi012.stdout:{"epoch":7,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:18:52.315442+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:18:53.033 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-08-31T08:18:54.067 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:18:54.066+0000 7f9dd0d77640 -1 osd.2 0 waiting for initial osdmap 2024-08-31T08:18:54.083 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:18:54.082+0000 7f9dc8382640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:18:54.115 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:18:54.114+0000 7f6c95c5a640 -1 osd.0 0 waiting for initial osdmap 2024-08-31T08:18:54.123 INFO:tasks.ceph.cluster1.osd.1.smithi012.stderr:2024-08-31T08:18:54.122+0000 7fcaa2071640 -1 osd.1 0 waiting for initial osdmap 2024-08-31T08:18:54.128 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:18:54.128+0000 7f6c8da79640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:18:54.141 INFO:tasks.ceph.cluster1.osd.1.smithi012.stderr:2024-08-31T08:18:54.141+0000 7fca99e90640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:18:59.336 DEBUG:teuthology.orchestra.run.smithi012:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:18:59.633 INFO:teuthology.misc.health.smithi012.stdout: 2024-08-31T08:18:59.634 INFO:teuthology.misc.health.smithi012.stdout:{"epoch":11,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:18:58.076182+0000","last_up_change":"2024-08-31T08:18:55.068524+0000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:18:55.330439+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6801","nonce":3876723986}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6803","nonce":3876723986}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6807","nonce":3876723986}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6805","nonce":3876723986}]},"public_addr":"172.21.15.12:6801/3876723986","cluster_addr":"172.21.15.12:6803/3876723986","heartbeat_back_addr":"172.21.15.12:6807/3876723986","heartbeat_front_addr":"172.21.15.12:6805/3876723986","state":["exists","up"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6817","nonce":1464980241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6819","nonce":1464980241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6823","nonce":1464980241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6821","nonce":1464980241}]},"public_addr":"172.21.15.12:6817/1464980241","cluster_addr":"172.21.15.12:6819/1464980241","heartbeat_back_addr":"172.21.15.12:6823/1464980241","heartbeat_front_addr":"172.21.15.12:6821/1464980241","state":["exists","up"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6809","nonce":3258870785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6811","nonce":3258870785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6815","nonce":3258870785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6813","nonce":3258870785}]},"public_addr":"172.21.15.12:6809/3258870785","cluster_addr":"172.21.15.12:6811/3258870785","heartbeat_back_addr":"172.21.15.12:6815/3258870785","heartbeat_front_addr":"172.21.15.12:6813/3258870785","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.017842+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.253258+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.227440+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:18:59.644 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-08-31T08:18:59.645 INFO:tasks.ceph:Creating RBD pool 2024-08-31T08:18:59.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo ceph --cluster cluster1 osd pool create rbd 8 2024-08-31T08:19:00.087 INFO:teuthology.orchestra.run.smithi012.stderr:pool 'rbd' created 2024-08-31T08:19:00.103 DEBUG:teuthology.orchestra.run.smithi012:> rbd --cluster cluster1 pool init rbd 2024-08-31T08:19:03.121 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2024-08-31T08:19:03.122 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2024-08-31T08:19:03.122 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-08-31T08:19:03.426 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:03.438 INFO:teuthology.orchestra.run.smithi012.stdout:[{"version":4,"timestamp":"2024-08-31T08:18:54.123229+0000","name":"","changes":[{"name":"osd.1/osd_mclock_max_capacity_iops_ssd","new_value":"4709.898770"}]}] 2024-08-31T08:19:03.438 INFO:tasks.ceph_manager:config epoch is 4 2024-08-31T08:19:03.438 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2024-08-31T08:19:03.439 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2024-08-31T08:19:03.439 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2024-08-31T08:19:03.784 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:03.795 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":3838828645},{"type":"v1","addr":"172.21.15.12:6825","nonce":3838828645}]},"active_addr":"172.21.15.12:6825/3838828645","active_change":"2024-08-31T08:18:49.309419+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3410020190}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3732563106}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2575817725}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2833896818}]}]} 2024-08-31T08:19:03.797 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2024-08-31T08:19:03.798 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2024-08-31T08:19:03.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:19:04.089 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:04.089 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":15,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:19:03.094056+0000","last_up_change":"2024-08-31T08:18:55.068524+0000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:18:55.330439+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-31T08:18:59.942792+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6801","nonce":3876723986}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6803","nonce":3876723986}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6807","nonce":3876723986}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6805","nonce":3876723986}]},"public_addr":"172.21.15.12:6801/3876723986","cluster_addr":"172.21.15.12:6803/3876723986","heartbeat_back_addr":"172.21.15.12:6807/3876723986","heartbeat_front_addr":"172.21.15.12:6805/3876723986","state":["exists","up"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6817","nonce":1464980241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6819","nonce":1464980241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6823","nonce":1464980241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6821","nonce":1464980241}]},"public_addr":"172.21.15.12:6817/1464980241","cluster_addr":"172.21.15.12:6819/1464980241","heartbeat_back_addr":"172.21.15.12:6823/1464980241","heartbeat_front_addr":"172.21.15.12:6821/1464980241","state":["exists","up"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6809","nonce":3258870785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6811","nonce":3258870785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6815","nonce":3258870785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6813","nonce":3258870785}]},"public_addr":"172.21.15.12:6809/3258870785","cluster_addr":"172.21.15.12:6811/3258870785","heartbeat_back_addr":"172.21.15.12:6815/3258870785","heartbeat_front_addr":"172.21.15.12:6813/3258870785","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.017842+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.253258+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.227440+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:04.102 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2024-08-31T08:19:04.103 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2024-08-31T08:19:04.396 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:04.396 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":15,"fsid":"c26ce5cc-1226-40b9-9482-3cb99b32adb2","created":"2024-08-31T08:18:44.840898+0000","modified":"2024-08-31T08:19:03.094056+0000","last_up_change":"2024-08-31T08:18:55.068524+0000","last_in_change":"2024-08-31T08:18:45.700368+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:18:55.330439+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-31T08:18:59.942792+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e6db31da-6057-4ef4-83ac-325ca3fc14c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6801","nonce":3876723986}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6803","nonce":3876723986}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6807","nonce":3876723986}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":3876723986},{"type":"v1","addr":"172.21.15.12:6805","nonce":3876723986}]},"public_addr":"172.21.15.12:6801/3876723986","cluster_addr":"172.21.15.12:6803/3876723986","heartbeat_back_addr":"172.21.15.12:6807/3876723986","heartbeat_front_addr":"172.21.15.12:6805/3876723986","state":["exists","up"]},{"osd":1,"uuid":"425794f8-f55e-44b1-a291-a3d71b3b00d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6817","nonce":1464980241}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6819","nonce":1464980241}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6823","nonce":1464980241}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":1464980241},{"type":"v1","addr":"172.21.15.12:6821","nonce":1464980241}]},"public_addr":"172.21.15.12:6817/1464980241","cluster_addr":"172.21.15.12:6819/1464980241","heartbeat_back_addr":"172.21.15.12:6823/1464980241","heartbeat_front_addr":"172.21.15.12:6821/1464980241","state":["exists","up"]},{"osd":2,"uuid":"09b2fb83-7065-4d81-9966-bee8cdc8886c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6809","nonce":3258870785}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6811","nonce":3258870785}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6815","nonce":3258870785}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":3258870785},{"type":"v1","addr":"172.21.15.12:6813","nonce":3258870785}]},"public_addr":"172.21.15.12:6809/3258870785","cluster_addr":"172.21.15.12:6811/3258870785","heartbeat_back_addr":"172.21.15.12:6815/3258870785","heartbeat_front_addr":"172.21.15.12:6813/3258870785","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.017842+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.253258+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:18:52.227440+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:04.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2024-08-31T08:19:04.409 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2024-08-31T08:19:04.410 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2024-08-31T08:19:04.549 INFO:teuthology.orchestra.run.smithi012.stdout:34359738374 2024-08-31T08:19:04.549 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-08-31T08:19:04.550 INFO:teuthology.orchestra.run.smithi012.stdout:34359738373 2024-08-31T08:19:04.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-08-31T08:19:04.551 INFO:teuthology.orchestra.run.smithi012.stdout:34359738374 2024-08-31T08:19:04.552 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-08-31T08:19:04.893 INFO:teuthology.orchestra.run.smithi012.stdout:34359738373 2024-08-31T08:19:04.897 INFO:teuthology.orchestra.run.smithi012.stdout:34359738372 2024-08-31T08:19:04.903 INFO:teuthology.orchestra.run.smithi012.stdout:34359738373 2024-08-31T08:19:04.907 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738373 for osd.0 2024-08-31T08:19:04.911 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738372 for osd.1 2024-08-31T08:19:04.916 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738373 for osd.2 2024-08-31T08:19:05.908 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2024-08-31T08:19:05.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2024-08-31T08:19:05.917 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2024-08-31T08:19:06.228 INFO:teuthology.orchestra.run.smithi012.stdout:34359738374 2024-08-31T08:19:06.241 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738374 for osd.0 2024-08-31T08:19:06.241 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:06.265 INFO:teuthology.orchestra.run.smithi012.stdout:34359738374 2024-08-31T08:19:06.266 INFO:teuthology.orchestra.run.smithi012.stdout:34359738373 2024-08-31T08:19:06.280 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738374 got 34359738374 for osd.2 2024-08-31T08:19:06.280 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:06.281 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738373 for osd.1 2024-08-31T08:19:06.281 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:06.281 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2024-08-31T08:19:06.281 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-08-31T08:19:06.609 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:06.610 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-08-31T08:19:06.621 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-08-31T08:19:05.315399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82644,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032876,"statfs":{"total":289910292480,"available":289825665024,"internally_reserved":0,"allocated":1589248,"data_stored":1295959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.227355"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105904+0000","last_change":"2024-08-31T08:19:03.106120+0000","last_active":"2024-08-31T08:19:03.105904+0000","last_peered":"2024-08-31T08:19:03.105904+0000","last_clean":"2024-08-31T08:19:03.105904+0000","last_became_active":"2024-08-31T08:19:01.096301+0000","last_became_peered":"2024-08-31T08:19:01.096301+0000","last_unstale":"2024-08-31T08:19:03.105904+0000","last_undegraded":"2024-08-31T08:19:03.105904+0000","last_fullsized":"2024-08-31T08:19:03.105904+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:42:11.874194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072929400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106179+0000","last_change":"2024-08-31T08:19:03.106423+0000","last_active":"2024-08-31T08:19:03.106179+0000","last_peered":"2024-08-31T08:19:03.106179+0000","last_clean":"2024-08-31T08:19:03.106179+0000","last_became_active":"2024-08-31T08:19:01.096091+0000","last_became_peered":"2024-08-31T08:19:01.096091+0000","last_unstale":"2024-08-31T08:19:03.106179+0000","last_undegraded":"2024-08-31T08:19:03.106179+0000","last_fullsized":"2024-08-31T08:19:03.106179+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:57:50.813338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010045259999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106066+0000","last_change":"2024-08-31T08:19:03.106282+0000","last_active":"2024-08-31T08:19:03.106066+0000","last_peered":"2024-08-31T08:19:03.106066+0000","last_clean":"2024-08-31T08:19:03.106066+0000","last_became_active":"2024-08-31T08:19:01.096118+0000","last_became_peered":"2024-08-31T08:19:01.096118+0000","last_unstale":"2024-08-31T08:19:03.106066+0000","last_undegraded":"2024-08-31T08:19:03.106066+0000","last_fullsized":"2024-08-31T08:19:03.106066+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:54:30.666353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082391699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105769+0000","last_change":"2024-08-31T08:19:03.106009+0000","last_active":"2024-08-31T08:19:03.105769+0000","last_peered":"2024-08-31T08:19:03.105769+0000","last_clean":"2024-08-31T08:19:03.105769+0000","last_became_active":"2024-08-31T08:19:01.096989+0000","last_became_peered":"2024-08-31T08:19:01.096989+0000","last_unstale":"2024-08-31T08:19:03.105769+0000","last_undegraded":"2024-08-31T08:19:03.105769+0000","last_fullsized":"2024-08-31T08:19:03.105769+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T09:26:18.343191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000644727,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.101686+0000","last_change":"2024-08-31T08:19:03.101686+0000","last_active":"2024-08-31T08:19:03.101686+0000","last_peered":"2024-08-31T08:19:03.101686+0000","last_clean":"2024-08-31T08:19:03.101686+0000","last_became_active":"2024-08-31T08:19:01.094324+0000","last_became_peered":"2024-08-31T08:19:01.094324+0000","last_unstale":"2024-08-31T08:19:03.101686+0000","last_undegraded":"2024-08-31T08:19:03.101686+0000","last_fullsized":"2024-08-31T08:19:03.101686+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:05:28.512807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097499299999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.098867+0000","last_change":"2024-08-31T08:19:03.099180+0000","last_active":"2024-08-31T08:19:03.098867+0000","last_peered":"2024-08-31T08:19:03.098867+0000","last_clean":"2024-08-31T08:19:03.098867+0000","last_became_active":"2024-08-31T08:19:01.095920+0000","last_became_peered":"2024-08-31T08:19:01.095920+0000","last_unstale":"2024-08-31T08:19:03.098867+0000","last_undegraded":"2024-08-31T08:19:03.098867+0000","last_fullsized":"2024-08-31T08:19:03.098867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T17:23:14.353550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054289799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.098905+0000","last_change":"2024-08-31T08:19:03.099197+0000","last_active":"2024-08-31T08:19:03.098905+0000","last_peered":"2024-08-31T08:19:03.098905+0000","last_clean":"2024-08-31T08:19:03.098905+0000","last_became_active":"2024-08-31T08:19:01.094677+0000","last_became_peered":"2024-08-31T08:19:01.094677+0000","last_unstale":"2024-08-31T08:19:03.098905+0000","last_undegraded":"2024-08-31T08:19:03.098905+0000","last_fullsized":"2024-08-31T08:19:03.098905+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T20:05:56.083507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053188300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106346+0000","last_change":"2024-08-31T08:19:03.106592+0000","last_active":"2024-08-31T08:19:03.106346+0000","last_peered":"2024-08-31T08:19:03.106346+0000","last_clean":"2024-08-31T08:19:03.106346+0000","last_became_active":"2024-08-31T08:19:01.095784+0000","last_became_peered":"2024-08-31T08:19:01.095784+0000","last_unstale":"2024-08-31T08:19:03.106346+0000","last_undegraded":"2024-08-31T08:19:03.106346+0000","last_fullsized":"2024-08-31T08:19:03.106346+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T15:56:26.298593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011242030000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105252+0000","last_change":"2024-08-31T08:18:58.072253+0000","last_active":"2024-08-31T08:19:03.105252+0000","last_peered":"2024-08-31T08:19:03.105252+0000","last_clean":"2024-08-31T08:19:03.105252+0000","last_became_active":"2024-08-31T08:18:58.071242+0000","last_became_peered":"2024-08-31T08:18:58.071242+0000","last_unstale":"2024-08-31T08:19:03.105252+0000","last_undegraded":"2024-08-31T08:19:03.105252+0000","last_fullsized":"2024-08-31T08:19:03.105252+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:18:56.069284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:18:56.069284+0000","last_clean_scrub_stamp":"2024-08-31T08:18:56.069284+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:31:49.523803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":131072,"data_stored":38395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T08:19:06.622 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2024-08-31T08:19:06.911 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:06.911 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-08-31T08:19:06.923 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2024-08-31T08:19:05.315399+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82644,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":81019,"kb_avail":283032876,"statfs":{"total":289910292480,"available":289825665024,"internally_reserved":0,"allocated":1589248,"data_stored":1295959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":82963791},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.227355"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105904+0000","last_change":"2024-08-31T08:19:03.106120+0000","last_active":"2024-08-31T08:19:03.105904+0000","last_peered":"2024-08-31T08:19:03.105904+0000","last_clean":"2024-08-31T08:19:03.105904+0000","last_became_active":"2024-08-31T08:19:01.096301+0000","last_became_peered":"2024-08-31T08:19:01.096301+0000","last_unstale":"2024-08-31T08:19:03.105904+0000","last_undegraded":"2024-08-31T08:19:03.105904+0000","last_fullsized":"2024-08-31T08:19:03.105904+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:42:11.874194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072929400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106179+0000","last_change":"2024-08-31T08:19:03.106423+0000","last_active":"2024-08-31T08:19:03.106179+0000","last_peered":"2024-08-31T08:19:03.106179+0000","last_clean":"2024-08-31T08:19:03.106179+0000","last_became_active":"2024-08-31T08:19:01.096091+0000","last_became_peered":"2024-08-31T08:19:01.096091+0000","last_unstale":"2024-08-31T08:19:03.106179+0000","last_undegraded":"2024-08-31T08:19:03.106179+0000","last_fullsized":"2024-08-31T08:19:03.106179+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:57:50.813338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010045259999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106066+0000","last_change":"2024-08-31T08:19:03.106282+0000","last_active":"2024-08-31T08:19:03.106066+0000","last_peered":"2024-08-31T08:19:03.106066+0000","last_clean":"2024-08-31T08:19:03.106066+0000","last_became_active":"2024-08-31T08:19:01.096118+0000","last_became_peered":"2024-08-31T08:19:01.096118+0000","last_unstale":"2024-08-31T08:19:03.106066+0000","last_undegraded":"2024-08-31T08:19:03.106066+0000","last_fullsized":"2024-08-31T08:19:03.106066+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:54:30.666353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082391699999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105769+0000","last_change":"2024-08-31T08:19:03.106009+0000","last_active":"2024-08-31T08:19:03.105769+0000","last_peered":"2024-08-31T08:19:03.105769+0000","last_clean":"2024-08-31T08:19:03.105769+0000","last_became_active":"2024-08-31T08:19:01.096989+0000","last_became_peered":"2024-08-31T08:19:01.096989+0000","last_unstale":"2024-08-31T08:19:03.105769+0000","last_undegraded":"2024-08-31T08:19:03.105769+0000","last_fullsized":"2024-08-31T08:19:03.105769+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T09:26:18.343191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000644727,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.101686+0000","last_change":"2024-08-31T08:19:03.101686+0000","last_active":"2024-08-31T08:19:03.101686+0000","last_peered":"2024-08-31T08:19:03.101686+0000","last_clean":"2024-08-31T08:19:03.101686+0000","last_became_active":"2024-08-31T08:19:01.094324+0000","last_became_peered":"2024-08-31T08:19:01.094324+0000","last_unstale":"2024-08-31T08:19:03.101686+0000","last_undegraded":"2024-08-31T08:19:03.101686+0000","last_fullsized":"2024-08-31T08:19:03.101686+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:05:28.512807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097499299999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.098867+0000","last_change":"2024-08-31T08:19:03.099180+0000","last_active":"2024-08-31T08:19:03.098867+0000","last_peered":"2024-08-31T08:19:03.098867+0000","last_clean":"2024-08-31T08:19:03.098867+0000","last_became_active":"2024-08-31T08:19:01.095920+0000","last_became_peered":"2024-08-31T08:19:01.095920+0000","last_unstale":"2024-08-31T08:19:03.098867+0000","last_undegraded":"2024-08-31T08:19:03.098867+0000","last_fullsized":"2024-08-31T08:19:03.098867+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T17:23:14.353550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054289799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.098905+0000","last_change":"2024-08-31T08:19:03.099197+0000","last_active":"2024-08-31T08:19:03.098905+0000","last_peered":"2024-08-31T08:19:03.098905+0000","last_clean":"2024-08-31T08:19:03.098905+0000","last_became_active":"2024-08-31T08:19:01.094677+0000","last_became_peered":"2024-08-31T08:19:01.094677+0000","last_unstale":"2024-08-31T08:19:03.098905+0000","last_undegraded":"2024-08-31T08:19:03.098905+0000","last_fullsized":"2024-08-31T08:19:03.098905+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T20:05:56.083507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053188300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.106346+0000","last_change":"2024-08-31T08:19:03.106592+0000","last_active":"2024-08-31T08:19:03.106346+0000","last_peered":"2024-08-31T08:19:03.106346+0000","last_clean":"2024-08-31T08:19:03.106346+0000","last_became_active":"2024-08-31T08:19:01.095784+0000","last_became_peered":"2024-08-31T08:19:01.095784+0000","last_unstale":"2024-08-31T08:19:03.106346+0000","last_undegraded":"2024-08-31T08:19:03.106346+0000","last_fullsized":"2024-08-31T08:19:03.106346+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:00.083379+0000","last_clean_scrub_stamp":"2024-08-31T08:19:00.083379+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T15:56:26.298593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011242030000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":88,"reported_epoch":15,"state":"active+clean","last_fresh":"2024-08-31T08:19:03.105252+0000","last_change":"2024-08-31T08:18:58.072253+0000","last_active":"2024-08-31T08:19:03.105252+0000","last_peered":"2024-08-31T08:19:03.105252+0000","last_clean":"2024-08-31T08:19:03.105252+0000","last_became_active":"2024-08-31T08:18:58.071242+0000","last_became_peered":"2024-08-31T08:18:58.071242+0000","last_unstale":"2024-08-31T08:19:03.105252+0000","last_undegraded":"2024-08-31T08:19:03.105252+0000","last_fullsized":"2024-08-31T08:19:03.105252+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:18:56.069284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:18:56.069284+0000","last_clean_scrub_stamp":"2024-08-31T08:18:56.069284+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T12:31:49.523803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27156,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344684,"statfs":{"total":96636764160,"available":96608956416,"internally_reserved":0,"allocated":131072,"data_stored":38395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738374,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27744,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344096,"statfs":{"total":96636764160,"available":96608354304,"internally_reserved":0,"allocated":729088,"data_stored":628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T08:19:06.924 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2024-08-31T08:19:06.924 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2024-08-31T08:19:06.924 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2024-08-31T08:19:06.924 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2024-08-31T08:19:07.255 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:07.255 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-31T08:19:07.268 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2024-08-31T08:19:07.268 INFO:teuthology.run_tasks:Running task ceph... 2024-08-31T08:19:07.280 INFO:tasks.ceph:Creating ceph cluster cluster2... 2024-08-31T08:19:07.280 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-08-31T08:19:07.281 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi/7883237', 'branch': 'wip-dis-testing', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid policy/none rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}', 'email': 'idryomov@gmail.com', 'first_in_suite': False, 'job_id': '7883237', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'dis-2024-08-31_07:23:49-rbd-wip-dis-testing-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-dis-testing'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-dis-testing', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, 'owner': 'scheduled_dis@teuthology', 'priority': 75, 'repo': 'https://github.com/ceph/ceph-ci', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 4871, 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'sleep_before_teardown': 0, 'subset': '98/100', 'suite': 'rbd', 'suite_branch': 'wip-dis-testing', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci', 'suite_sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760', 'targets': {'smithi012.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNEx9XsXup32GoJWsUuPxH1n47mKHEI6yaBmm41mtD7h8aNmNH7iD1ARqK8OBx22xCY1NFPDkU2kwMBJEqjSn7E=', 'smithi167.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpbZp5xuE44w4oUgvkpwPbPMuNfddfjlOsUe5biVwgLuxRVcedQHT4m1bf7uePACsma12cDwmAwkJhAgE8y+p8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'dc9f1ccbfcab74acda17f39d6ab018266071e760'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-08-31_07:23:49', 'tube': 'smithi', 'user': 'dis', 'verbose': True, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856'} 2024-08-31T08:19:07.281 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-08-31T08:19:07.284 DEBUG:teuthology.orchestra.run.smithi167:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2024-08-31T08:19:07.302 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m0777 -- /var/run/ceph 2024-08-31T08:19:07.339 DEBUG:teuthology.orchestra.run.smithi167:> sudo install -d -m0777 -- /var/run/ceph 2024-08-31T08:19:07.372 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:07.373 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/scratch_devs of=/dev/stdout 2024-08-31T08:19:07.431 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-31T08:19:07.431 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_1 2024-08-31T08:19:07.488 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2024-08-31 08:18:23.580619956 +0000 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2024-08-31 08:15:45.161616917 +0000 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2024-08-31 08:15:45.161616917 +0000 2024-08-31T08:19:07.489 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: 2024-08-31 08:15:45.161616917 +0000 2024-08-31T08:19:07.489 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-31T08:19:07.554 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2024-08-31T08:19:07.555 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2024-08-31T08:19:07.555 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000277666 s, 1.8 MB/s 2024-08-31T08:19:07.556 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-31T08:19:07.613 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_2 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2024-08-31 08:18:23.581619937 +0000 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2024-08-31 08:15:45.492610487 +0000 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2024-08-31 08:15:45.492610487 +0000 2024-08-31T08:19:07.671 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: 2024-08-31 08:15:45.492610487 +0000 2024-08-31T08:19:07.671 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-31T08:19:07.736 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2024-08-31T08:19:07.736 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2024-08-31T08:19:07.736 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000188337 s, 2.7 MB/s 2024-08-31T08:19:07.738 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-31T08:19:07.796 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_3 2024-08-31T08:19:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-31T08:19:07.852 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2024-08-31 08:18:23.581619937 +0000 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2024-08-31 08:15:45.815604212 +0000 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2024-08-31 08:15:45.815604212 +0000 2024-08-31T08:19:07.853 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: 2024-08-31 08:15:45.815604212 +0000 2024-08-31T08:19:07.854 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-31T08:19:07.916 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2024-08-31T08:19:07.917 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2024-08-31T08:19:07.917 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000186504 s, 2.7 MB/s 2024-08-31T08:19:07.918 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-31T08:19:07.974 DEBUG:teuthology.orchestra.run.smithi167:> stat /dev/vg_nvme/lv_4 2024-08-31T08:19:08.030 INFO:teuthology.orchestra.run.smithi167.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-31T08:19:08.030 INFO:teuthology.orchestra.run.smithi167.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-31T08:19:08.030 INFO:teuthology.orchestra.run.smithi167.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-08-31T08:19:08.030 INFO:teuthology.orchestra.run.smithi167.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-31T08:19:08.030 INFO:teuthology.orchestra.run.smithi167.stdout:Context: system_u:object_r:device_t:s0 2024-08-31T08:19:08.031 INFO:teuthology.orchestra.run.smithi167.stdout:Access: 2024-08-31 08:18:23.581619937 +0000 2024-08-31T08:19:08.031 INFO:teuthology.orchestra.run.smithi167.stdout:Modify: 2024-08-31 08:15:46.139597918 +0000 2024-08-31T08:19:08.031 INFO:teuthology.orchestra.run.smithi167.stdout:Change: 2024-08-31 08:15:46.139597918 +0000 2024-08-31T08:19:08.031 INFO:teuthology.orchestra.run.smithi167.stdout: Birth: 2024-08-31 08:15:46.139597918 +0000 2024-08-31T08:19:08.031 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-31T08:19:08.094 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records in 2024-08-31T08:19:08.094 INFO:teuthology.orchestra.run.smithi167.stderr:1+0 records out 2024-08-31T08:19:08.095 INFO:teuthology.orchestra.run.smithi167.stderr:512 bytes copied, 0.000178885 s, 2.9 MB/s 2024-08-31T08:19:08.096 DEBUG:teuthology.orchestra.run.smithi167:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-31T08:19:08.153 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:19:08.154 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi167.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-08-31T08:19:08.154 INFO:tasks.ceph:Generating config... 2024-08-31T08:19:08.156 INFO:tasks.ceph:[client] debug rbd = 20 2024-08-31T08:19:08.156 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2024-08-31T08:19:08.156 INFO:tasks.ceph:[client] log to stderr = False 2024-08-31T08:19:08.156 INFO:tasks.ceph:[client] rbd default features = 125 2024-08-31T08:19:08.156 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2024-08-31T08:19:08.157 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.157 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.157 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.157 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.157 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.158 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.158 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.158 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.158 INFO:tasks.ceph:[global] mon client directed command retry = 5 2024-08-31T08:19:08.158 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-08-31T08:19:08.158 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2024-08-31T08:19:08.159 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-08-31T08:19:08.159 INFO:tasks.ceph:[mgr] debug ms = 1 2024-08-31T08:19:08.159 INFO:tasks.ceph:[mon] debug mon = 20 2024-08-31T08:19:08.159 INFO:tasks.ceph:[mon] debug ms = 1 2024-08-31T08:19:08.159 INFO:tasks.ceph:[mon] debug paxos = 20 2024-08-31T08:19:08.159 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-08-31T08:19:08.160 INFO:tasks.ceph:[osd] debug ms = 1 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] debug osd = 20 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-08-31T08:19:08.161 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-08-31T08:19:08.162 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-08-31T08:19:08.162 INFO:tasks.ceph:Setting up cluster2.mon.a... 2024-08-31T08:19:08.162 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.234 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.237 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.320 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.386 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '172.21.15.167')] 2024-08-31T08:19:08.387 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.167', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2024-08-31T08:19:08.387 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:08.387 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-08-31T08:19:08.442 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.167 --print /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:monmaptool: generated fsid c9c7e778-1922-4472-806b-f799b2b74be3 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:setting min_mon_release = quincy 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:epoch 0 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:fsid c9c7e778-1922-4472-806b-f799b2b74be3 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:last_changed 2024-08-31T08:19:08.522848+0000 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:created 2024-08-31T08:19:08.522848+0000 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:min_mon_release 17 (quincy) 2024-08-31T08:19:08.523 INFO:teuthology.orchestra.run.smithi167.stdout:election_strategy: 1 2024-08-31T08:19:08.524 INFO:teuthology.orchestra.run.smithi167.stdout:0: [v2:172.21.15.167:3300/0,v1:172.21.15.167:6789/0] mon.a 2024-08-31T08:19:08.524 INFO:teuthology.orchestra.run.smithi167.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2024-08-31T08:19:08.525 DEBUG:teuthology.orchestra.run.smithi167:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-08-31T08:19:08.580 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID c9c7e778-1922-4472-806b-f799b2b74be3... 2024-08-31T08:19:08.581 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-08-31T08:19:08.583 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2024-08-31T08:19:08.631 INFO:teuthology.orchestra.run.smithi012.stdout:[global] 2024-08-31T08:19:08.631 INFO:teuthology.orchestra.run.smithi012.stdout: chdir = "" 2024-08-31T08:19:08.631 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: auth supported = cephx 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: filestore xattr use omap = true 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: mon clock drift allowed = 1.000 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: osd crush chooseleaf type = 0 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: auth debug = true 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: ms die on old message = true 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: ms die on bug = true 2024-08-31T08:19:08.632 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon pg warn max object skew = 0 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: # disable pg_autoscaler by default for new pools 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: osd pool default size = 2 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd allow primary affinity = true 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd allow pg remap = true 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on legacy crush tunables = false 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on crush straw calc version zero = false 2024-08-31T08:19:08.633 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on no sortbitwise = false 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on osd down out interval zero = false 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on too few osds = false 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon_allow_pool_size_one = true 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: osd default data pool replay window = 5 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon allow pool delete = true 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.634 INFO:teuthology.orchestra.run.smithi012.stdout: mon cluster log file level = debug 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: debug asserts on shutdown = true 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: mon health detail to clog = false 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: mon host = 172.21.15.167 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: mon client directed command retry = 5 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: mon warn on pool no app = False 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: ms inject socket failures = 5000 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: fsid = c9c7e778-1922-4472-806b-f799b2b74be3 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout:[osd] 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: osd journal size = 100 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: osd scrub load threshold = 5.0 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: osd scrub max interval = 600 2024-08-31T08:19:08.635 INFO:teuthology.orchestra.run.smithi012.stdout: osd mclock profile = high_recovery_ops 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd recover clone overlap = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd recovery max chunk = 1048576 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug shutdown = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug op order = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug verify stray on activate = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug trim objects = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd open classes on start = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug pg log writeout = true 2024-08-31T08:19:08.636 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: osd deep scrub update digest min age = 30 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: osd map max advance = 10 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: journal zero on create = true 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: filestore ondisk finisher threads = 3 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: filestore apply finisher threads = 3 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: bdev debug aio = true 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: osd debug misdirected ops = true 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: bluefs allocator = hybrid 2024-08-31T08:19:08.637 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore allocator = hybrid 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore block size = 96636764160 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: bluestore fsck on mount = True 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: debug bluefs = 1/20 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: debug bluestore = 1/20 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: debug osd = 20 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: debug rocksdb = 4/10 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd backfillfull_ratio = 0.85 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd full ratio = 0.9 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd nearfull ratio = 0.8 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: osd failsafe full ratio = 0.95 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: osd objectstore = bluestore 2024-08-31T08:19:08.638 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout:[mgr] 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug mgr = 20 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug mon = 20 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug auth = 20 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: mgr/telemetry/nag = false 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout:[mon] 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug ms = 1 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug mon = 20 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug paxos = 20 2024-08-31T08:19:08.639 INFO:teuthology.orchestra.run.smithi012.stdout: debug auth = 20 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon data avail warn = 5 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon mgr mkfs grace = 240 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd reporter subtree level = osd 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon osd prime pg temp = true 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: auth mon ticket ttl = 660 # 11m 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: auth service ticket ttl = 240 # 4m 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: # don't complain about insecure global_id in the test suite 2024-08-31T08:19:08.640 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: # 1m isn't quite enough 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: mon_down_mkfs_grace = 2m 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: mon_warn_on_filestore_osds = false 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout:[client] 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: rgw cache enabled = true 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: rgw enable ops log = true 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: rgw enable usage log = true 2024-08-31T08:19:08.641 INFO:teuthology.orchestra.run.smithi012.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: debug rbd = 20 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: debug rbd_mirror = 15 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: log to stderr = False 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: rbd default features = 125 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: rbd mirror image policy type = none 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout:[mon.a] 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.0] 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.1] 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.642 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.2] 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout:[client.mirror.3] 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.643 INFO:teuthology.orchestra.run.smithi012.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.662 INFO:teuthology.orchestra.run.smithi167.stdout:[global] 2024-08-31T08:19:08.662 INFO:teuthology.orchestra.run.smithi167.stdout: chdir = "" 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: auth supported = cephx 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: filestore xattr use omap = true 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: mon clock drift allowed = 1.000 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: osd crush chooseleaf type = 0 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: auth debug = true 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: ms die on old message = true 2024-08-31T08:19:08.663 INFO:teuthology.orchestra.run.smithi167.stdout: ms die on bug = true 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon max pg per osd = 10000 # >= luminous 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon pg warn max object skew = 0 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: # disable pg_autoscaler by default for new pools 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: osd_pool_default_pg_autoscale_mode = off 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: osd pool default size = 2 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd allow primary affinity = true 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd allow pg remap = true 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on legacy crush tunables = false 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on crush straw calc version zero = false 2024-08-31T08:19:08.664 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on no sortbitwise = false 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on osd down out interval zero = false 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on too few osds = false 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_pool_no_redundancy = false 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon_allow_pool_size_one = true 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: osd default data pool replay window = 5 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon allow pool delete = true 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.665 INFO:teuthology.orchestra.run.smithi167.stdout: mon cluster log file level = debug 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: debug asserts on shutdown = true 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: mon health detail to clog = false 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: mon host = 172.21.15.167 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: mon client directed command retry = 5 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: mon warn on pool no app = False 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: ms inject socket failures = 5000 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: fsid = c9c7e778-1922-4472-806b-f799b2b74be3 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout:[osd] 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: osd journal size = 100 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: osd scrub load threshold = 5.0 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: osd scrub max interval = 600 2024-08-31T08:19:08.666 INFO:teuthology.orchestra.run.smithi167.stdout: osd mclock profile = high_recovery_ops 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd recover clone overlap = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd recovery max chunk = 1048576 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug shutdown = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug op order = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug verify stray on activate = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug trim objects = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd open classes on start = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug pg log writeout = true 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: osd deep scrub update digest min age = 30 2024-08-31T08:19:08.667 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: osd map max advance = 10 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: journal zero on create = true 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: filestore ondisk finisher threads = 3 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: filestore apply finisher threads = 3 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: bdev debug aio = true 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: osd debug misdirected ops = true 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: bluefs allocator = hybrid 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore allocator = hybrid 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore block size = 96636764160 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: bluestore fsck on mount = True 2024-08-31T08:19:08.668 INFO:teuthology.orchestra.run.smithi167.stdout: debug bluefs = 1/20 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug bluestore = 1/20 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug osd = 20 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug rocksdb = 4/10 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd backfillfull_ratio = 0.85 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd full ratio = 0.9 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd nearfull ratio = 0.8 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: osd failsafe full ratio = 0.95 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: osd objectstore = bluestore 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout:[mgr] 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:19:08.669 INFO:teuthology.orchestra.run.smithi167.stdout: debug mgr = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug mon = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug auth = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mgr/telemetry/nag = false 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout:[mon] 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug ms = 1 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug mon = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug paxos = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: debug auth = 20 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon data avail warn = 5 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon mgr mkfs grace = 240 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min pgs per osd = 4 2024-08-31T08:19:08.670 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd reporter subtree level = osd 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: mon osd prime pg temp = true 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: mon reweight min bytes per osd = 10 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: auth mon ticket ttl = 660 # 11m 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: auth service ticket ttl = 240 # 4m 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: # don't complain about insecure global_id in the test suite 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: # 1m isn't quite enough 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: mon_down_mkfs_grace = 2m 2024-08-31T08:19:08.671 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: mon_warn_on_filestore_osds = false 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout:[client] 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: rgw cache enabled = true 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: rgw enable ops log = true 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: rgw enable usage log = true 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: debug rbd = 20 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: debug rbd_mirror = 15 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: log to stderr = False 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: rbd default features = 125 2024-08-31T08:19:08.672 INFO:teuthology.orchestra.run.smithi167.stdout: rbd mirror image policy type = none 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout:[mon.a] 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.0] 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.1] 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.2] 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.3] 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2024-08-31T08:19:08.673 INFO:teuthology.orchestra.run.smithi167.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2024-08-31T08:19:08.674 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2024-08-31T08:19:08.674 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.769 INFO:tasks.ceph:Copying monmap to all nodes... 2024-08-31T08:19:08.769 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:08.769 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2024-08-31T08:19:08.786 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:08.786 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2024-08-31T08:19:08.841 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi012.front.sepia.ceph.com 2024-08-31T08:19:08.841 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:19:08.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/cluster2.keyring 2024-08-31T08:19:08.841 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.881 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:19:08.881 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:08.940 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:19:08.940 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:08.940 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd of=/etc/ceph/cluster2.keyring 2024-08-31T08:19:08.940 DEBUG:teuthology.orchestra.run.smithi167:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2024-08-31T08:19:08.975 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:08.975 DEBUG:teuthology.orchestra.run.smithi167:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:09.031 INFO:tasks.ceph:Setting up mon nodes... 2024-08-31T08:19:09.031 INFO:tasks.ceph:Setting up mgr nodes... 2024-08-31T08:19:09.031 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2024-08-31T08:19:09.088 INFO:teuthology.orchestra.run.smithi012.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2024-08-31T08:19:09.091 INFO:tasks.ceph:Setting up mds nodes... 2024-08-31T08:19:09.091 INFO:tasks.ceph_client:Setting up client nodes... 2024-08-31T08:19:09.091 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2024-08-31T08:19:09.134 INFO:teuthology.orchestra.run.smithi012.stdout:creating /etc/ceph/cluster2.client.0.keyring 2024-08-31T08:19:09.148 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2024-08-31T08:19:09.189 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2024-08-31T08:19:09.202 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2024-08-31T08:19:09.283 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2024-08-31T08:19:09.296 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2024-08-31T08:19:09.377 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2024-08-31T08:19:09.389 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2024-08-31T08:19:09.471 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2024-08-31T08:19:09.483 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2024-08-31T08:19:09.564 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2024-08-31T08:19:09.577 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2024-08-31T08:19:09.659 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2024-08-31T08:19:09.671 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2024-08-31T08:19:09.751 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2024-08-31T08:19:09.764 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2024-08-31T08:19:09.844 INFO:teuthology.orchestra.run.smithi167.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2024-08-31T08:19:09.857 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-08-31T08:19:09.857 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi012.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@smithi167.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2024-08-31T08:19:09.857 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2024-08-31T08:19:09.921 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:19:09.921 INFO:tasks.ceph:role: cluster2.osd.0 2024-08-31T08:19:09.921 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:19:09.921 DEBUG:teuthology.orchestra.run.smithi167:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-08-31T08:19:09.993 INFO:teuthology.orchestra.run.smithi167.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:19:09.994 INFO:teuthology.orchestra.run.smithi167.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:19:10.505 INFO:teuthology.orchestra.run.smithi167.stdout:Discarding blocks...Done. 2024-08-31T08:19:10.507 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi167.front.sepia.ceph.com -o noatime 2024-08-31T08:19:10.507 DEBUG:teuthology.orchestra.run.smithi167:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2024-08-31T08:19:10.611 DEBUG:teuthology.orchestra.run.smithi167:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2024-08-31T08:19:10.641 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2024-08-31T08:19:10.706 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:19:10.706 INFO:tasks.ceph:role: cluster2.osd.1 2024-08-31T08:19:10.706 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:19:10.707 DEBUG:teuthology.orchestra.run.smithi167:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:19:10.780 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:19:10.781 INFO:teuthology.orchestra.run.smithi167.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:19:11.246 INFO:teuthology.orchestra.run.smithi167.stdout:Discarding blocks...Done. 2024-08-31T08:19:11.248 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi167.front.sepia.ceph.com -o noatime 2024-08-31T08:19:11.248 DEBUG:teuthology.orchestra.run.smithi167:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2024-08-31T08:19:11.331 DEBUG:teuthology.orchestra.run.smithi167:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2024-08-31T08:19:11.401 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2024-08-31T08:19:11.467 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2024-08-31T08:19:11.467 INFO:tasks.ceph:role: cluster2.osd.2 2024-08-31T08:19:11.467 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:19:11.468 DEBUG:teuthology.orchestra.run.smithi167:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 attr=2, projid32bit=1 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout: = sunit=0 swidth=0 blks 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-08-31T08:19:11.540 INFO:teuthology.orchestra.run.smithi167.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-08-31T08:19:11.541 INFO:teuthology.orchestra.run.smithi167.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-08-31T08:19:11.541 INFO:teuthology.orchestra.run.smithi167.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-08-31T08:19:12.069 INFO:teuthology.orchestra.run.smithi167.stdout:Discarding blocks...Done. 2024-08-31T08:19:12.071 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi167.front.sepia.ceph.com -o noatime 2024-08-31T08:19:12.071 DEBUG:teuthology.orchestra.run.smithi167:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2024-08-31T08:19:12.224 DEBUG:teuthology.orchestra.run.smithi167:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2024-08-31T08:19:12.256 DEBUG:teuthology.orchestra.run.smithi167:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:12.356 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:12.355+0000 7f97dad19640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2024-08-31T08:19:12.359 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:12.358+0000 7f97dad19640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2024-08-31T08:19:12.359 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:12.358+0000 7f97dad19640 -1 bdev(0x55ff3390f800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2024-08-31T08:19:12.360 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:12.358+0000 7f97dad19640 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2024-08-31T08:19:15.519 DEBUG:teuthology.orchestra.run.smithi167:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2024-08-31T08:19:15.546 DEBUG:teuthology.orchestra.run.smithi167:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:15.633 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:15.631+0000 7f4c5354d640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2024-08-31T08:19:15.633 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:15.632+0000 7f4c5354d640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2024-08-31T08:19:15.633 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:15.632+0000 7f4c5354d640 -1 bdev(0x55b2176a7800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2024-08-31T08:19:15.633 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:15.632+0000 7f4c5354d640 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2024-08-31T08:19:18.789 DEBUG:teuthology.orchestra.run.smithi167:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2024-08-31T08:19:18.815 DEBUG:teuthology.orchestra.run.smithi167:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:18.901 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:18.899+0000 7f0cbc80e640 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2024-08-31T08:19:18.901 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:18.900+0000 7f0cbc80e640 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2024-08-31T08:19:18.901 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:18.900+0000 7f0cbc80e640 -1 bdev(0x559c43555800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2024-08-31T08:19:18.901 INFO:teuthology.orchestra.run.smithi167.stderr:2024-08-31T08:19:18.900+0000 7f0cbc80e640 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2024-08-31T08:19:21.803 DEBUG:teuthology.orchestra.run.smithi167:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2024-08-31T08:19:21.829 INFO:tasks.ceph:Reading keys from all nodes... 2024-08-31T08:19:21.829 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:19:21.829 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2024-08-31T08:19:21.860 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:21.860 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2024-08-31T08:19:21.895 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:21.895 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2024-08-31T08:19:21.961 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:21.962 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2024-08-31T08:19:22.026 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-08-31T08:19:22.027 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2024-08-31T08:19:22.045 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.045 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2024-08-31T08:19:22.082 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.082 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2024-08-31T08:19:22.138 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.138 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2024-08-31T08:19:22.193 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.193 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2024-08-31T08:19:22.248 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.248 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2024-08-31T08:19:22.303 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.304 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2024-08-31T08:19:22.358 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.359 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2024-08-31T08:19:22.414 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:22.414 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2024-08-31T08:19:22.468 INFO:tasks.ceph:Adding keys to all mons... 2024-08-31T08:19:22.469 DEBUG:teuthology.orchestra.run.smithi167:> sudo tee -a /etc/ceph/cluster2.keyring 2024-08-31T08:19:22.531 INFO:teuthology.orchestra.run.smithi167.stdout:[mgr.x] 2024-08-31T08:19:22.531 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmmhJEBRAA09ed2NYOEALDD5K5RsZpng== 2024-08-31T08:19:22.531 INFO:teuthology.orchestra.run.smithi167.stdout:[osd.0] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQAA0tJmUBlAFRAAPRxgsjgUoro/W79Tw3++UQ== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[osd.1] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQAD0tJm0XS2JRAAgfDG18SRkj9OqICs9hJ1Ow== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[osd.2] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQAG0tJmrLCvNRAAL0VDDBE6YEk8+910alsC4A== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[client.0] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmO4sICBAAlcCZbwGvk5Lc7mWtpZKgYg== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmcZZJCxAAwN8G66HmpmipVSPYEDR3dg== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.0] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmx+nlEBAAP/sOoja7Kcr6uPO5mXgV5w== 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.1] 2024-08-31T08:19:22.532 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJm/8N4FhAAs8RBI8IDZ8jGj60BOlSnsw== 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.2] 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmH98UHBAAkCOzZ36QMkfR8VWjMZMxZA== 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.3] 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmPvWoIRAA7e9aQ4SpPdzYLSRpAdN5kw== 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.4] 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmZ11FJxAAzuQTbZ/vGpTR8lltHGr04A== 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.5] 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmyLTNLBAAOpxwy0DX3GJ3cNT7b1e7og== 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.6] 2024-08-31T08:19:22.533 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmjHlaMhAAKHJO3uM5z6S1NoJnqBk2PQ== 2024-08-31T08:19:22.534 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-08-31T08:19:22.626 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:19:22.677 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:19:22.728 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-08-31T08:19:22.779 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:22.834 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:22.888 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:22.942 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:22.996 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:23.050 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:23.104 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:23.158 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:23.211 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2024-08-31T08:19:23.265 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-08-31T08:19:23.266 DEBUG:teuthology.orchestra.run.smithi167:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2024-08-31T08:19:23.290 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2024-08-31T08:19:23.386 DEBUG:teuthology.orchestra.run.smithi167:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2024-08-31T08:19:23.410 DEBUG:teuthology.orchestra.run.smithi167:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2024-08-31T08:19:23.466 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2024-08-31T08:19:23.466 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2024-08-31T08:19:23.466 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2024-08-31T08:19:23.508 INFO:tasks.ceph.cluster2.mon.a:Started 2024-08-31T08:19:23.508 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2024-08-31T08:19:23.509 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2024-08-31T08:19:23.509 DEBUG:teuthology.orchestra.run.smithi012:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2024-08-31T08:19:23.511 INFO:tasks.ceph.cluster2.mgr.x:Started 2024-08-31T08:19:23.511 DEBUG:tasks.ceph:set 0 configs 2024-08-31T08:19:23.511 INFO:tasks.ceph:Setting crush tunables to default 2024-08-31T08:19:23.512 DEBUG:teuthology.orchestra.run.smithi167:> sudo ceph --cluster cluster2 osd crush tunables default 2024-08-31T08:19:23.681 INFO:teuthology.orchestra.run.smithi167.stderr:adjusted tunables profile to default 2024-08-31T08:19:23.692 INFO:tasks.ceph:check_enable_crimson: False 2024-08-31T08:19:23.692 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2024-08-31T08:19:23.692 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:23.692 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2024-08-31T08:19:23.717 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:23.717 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2024-08-31T08:19:23.782 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:19:23.782 DEBUG:teuthology.orchestra.run.smithi167:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2024-08-31T08:19:23.848 DEBUG:teuthology.orchestra.run.smithi167:> sudo ceph --cluster cluster2 osd new 4272991b-2bd9-4b65-a7fc-c4f6e5d05c75 0 2024-08-31T08:19:24.061 INFO:teuthology.orchestra.run.smithi167.stdout:0 2024-08-31T08:19:24.072 DEBUG:teuthology.orchestra.run.smithi167:> sudo ceph --cluster cluster2 osd new 1b93f235-4e86-4c42-8601-b05b5b777cac 1 2024-08-31T08:19:24.246 INFO:teuthology.orchestra.run.smithi167.stdout:1 2024-08-31T08:19:24.257 DEBUG:teuthology.orchestra.run.smithi167:> sudo ceph --cluster cluster2 osd new 92b6bc52-ef47-41b3-8c2e-c7ff34518eb7 2 2024-08-31T08:19:24.430 INFO:teuthology.orchestra.run.smithi167.stdout:2 2024-08-31T08:19:24.441 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2024-08-31T08:19:24.441 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2024-08-31T08:19:24.444 INFO:tasks.ceph.cluster2.osd.0:Started 2024-08-31T08:19:24.444 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2024-08-31T08:19:24.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2024-08-31T08:19:24.446 INFO:tasks.ceph.cluster2.osd.1:Started 2024-08-31T08:19:24.446 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2024-08-31T08:19:24.446 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2024-08-31T08:19:24.448 INFO:tasks.ceph.cluster2.osd.2:Started 2024-08-31T08:19:24.448 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:24.619 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:24.619 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":5,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:24.429956+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:24.630 INFO:tasks.ceph.ceph_manager.cluster2:[] 2024-08-31T08:19:24.630 INFO:tasks.ceph:Waiting for OSDs to come up 2024-08-31T08:19:24.932 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:25.091 INFO:teuthology.misc.health.smithi167.stdout: 2024-08-31T08:19:25.091 INFO:teuthology.misc.health.smithi167.stdout:{"epoch":5,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:24.429956+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:25.100 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-08-31T08:19:25.642 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:19:25.641+0000 7fb3245dd640 -1 Falling back to public interface 2024-08-31T08:19:25.874 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:19:25.873+0000 7f5b0bfd4640 -1 Falling back to public interface 2024-08-31T08:19:26.118 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:19:26.117+0000 7f6cc09b2640 -1 Falling back to public interface 2024-08-31T08:19:28.138 INFO:tasks.ceph.cluster2.mgr.x.smithi012.stderr:2024-08-31T08:19:28.136+0000 7ffa746c6640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-31T08:19:29.431 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:19:29.430+0000 7fb3245dd640 -1 osd.0 0 log_to_monitors true 2024-08-31T08:19:29.637 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:19:29.636+0000 7f6cc09b2640 -1 osd.2 0 log_to_monitors true 2024-08-31T08:19:30.181 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:19:30.180+0000 7f5b0bfd4640 -1 osd.1 0 log_to_monitors true 2024-08-31T08:19:31.404 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:31.740 INFO:teuthology.misc.health.smithi167.stdout: 2024-08-31T08:19:31.740 INFO:teuthology.misc.health.smithi167.stdout:{"epoch":7,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:31.136164+0000","last_up_change":"0.000000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:31.752 DEBUG:teuthology.misc:0 of 3 OSDs are up 2024-08-31T08:19:32.671 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:19:32.670+0000 7f6cbd155640 -1 osd.2 0 waiting for initial osdmap 2024-08-31T08:19:32.676 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:19:32.673+0000 7fb320d80640 -1 osd.0 0 waiting for initial osdmap 2024-08-31T08:19:32.694 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:19:32.693+0000 7fb31838b640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:19:32.694 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:19:32.693+0000 7f6cb4760640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:19:33.134 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:19:33.133+0000 7f5b07f63640 -1 osd.1 0 waiting for initial osdmap 2024-08-31T08:19:33.153 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:19:33.152+0000 7f5affd82640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-31T08:19:38.056 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:38.344 INFO:teuthology.misc.health.smithi167.stdout: 2024-08-31T08:19:38.344 INFO:teuthology.misc.health.smithi167.stdout:{"epoch":12,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:37.160539+0000","last_up_change":"2024-08-31T08:19:33.677518+0000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:19:35.151358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6801","nonce":3684510860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6803","nonce":3684510860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6807","nonce":3684510860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6805","nonce":3684510860}]},"public_addr":"172.21.15.167:6801/3684510860","cluster_addr":"172.21.15.167:6803/3684510860","heartbeat_back_addr":"172.21.15.167:6807/3684510860","heartbeat_front_addr":"172.21.15.167:6805/3684510860","state":["exists","up"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6809","nonce":1387293444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6811","nonce":1387293444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6815","nonce":1387293444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6813","nonce":1387293444}]},"public_addr":"172.21.15.167:6809/1387293444","cluster_addr":"172.21.15.167:6811/1387293444","heartbeat_back_addr":"172.21.15.167:6815/1387293444","heartbeat_front_addr":"172.21.15.167:6813/1387293444","state":["exists","up"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6817","nonce":468601409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6819","nonce":468601409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6823","nonce":468601409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6821","nonce":468601409}]},"public_addr":"172.21.15.167:6817/468601409","cluster_addr":"172.21.15.167:6819/468601409","heartbeat_back_addr":"172.21.15.167:6823/468601409","heartbeat_front_addr":"172.21.15.167:6821/468601409","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.433172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:31.159802+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.624825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:38.356 DEBUG:teuthology.misc:3 of 3 OSDs are up 2024-08-31T08:19:38.356 INFO:tasks.ceph:Creating RBD pool 2024-08-31T08:19:38.356 DEBUG:teuthology.orchestra.run.smithi167:> sudo ceph --cluster cluster2 osd pool create rbd 8 2024-08-31T08:19:39.171 INFO:teuthology.orchestra.run.smithi167.stderr:pool 'rbd' created 2024-08-31T08:19:39.188 DEBUG:teuthology.orchestra.run.smithi167:> rbd --cluster cluster2 pool init rbd 2024-08-31T08:19:42.194 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2024-08-31T08:19:42.195 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2024-08-31T08:19:42.196 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-08-31T08:19:42.494 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:42.506 INFO:teuthology.orchestra.run.smithi167.stdout:[{"version":4,"timestamp":"2024-08-31T08:19:33.134751+0000","name":"","changes":[{"name":"osd.1/osd_mclock_max_capacity_iops_ssd","new_value":"6212.083847"}]}] 2024-08-31T08:19:42.506 INFO:tasks.ceph_manager:config epoch is 4 2024-08-31T08:19:42.506 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2024-08-31T08:19:42.506 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2024-08-31T08:19:42.507 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2024-08-31T08:19:42.838 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:42.849 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":5,"flags":0,"active_gid":4098,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":207265633},{"type":"v1","addr":"172.21.15.12:6827","nonce":207265633}]},"active_addr":"172.21.15.12:6827/207265633","active_change":"2024-08-31T08:19:27.129551+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":3435435614}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2973886726}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":132008539}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2295318131}]}]} 2024-08-31T08:19:42.852 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2024-08-31T08:19:42.852 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2024-08-31T08:19:42.852 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:43.134 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:43.134 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":16,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:42.177055+0000","last_up_change":"2024-08-31T08:19:33.677518+0000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:19:35.151358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-31T08:19:38.645516+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6801","nonce":3684510860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6803","nonce":3684510860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6807","nonce":3684510860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6805","nonce":3684510860}]},"public_addr":"172.21.15.167:6801/3684510860","cluster_addr":"172.21.15.167:6803/3684510860","heartbeat_back_addr":"172.21.15.167:6807/3684510860","heartbeat_front_addr":"172.21.15.167:6805/3684510860","state":["exists","up"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6809","nonce":1387293444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6811","nonce":1387293444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6815","nonce":1387293444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6813","nonce":1387293444}]},"public_addr":"172.21.15.167:6809/1387293444","cluster_addr":"172.21.15.167:6811/1387293444","heartbeat_back_addr":"172.21.15.167:6815/1387293444","heartbeat_front_addr":"172.21.15.167:6813/1387293444","state":["exists","up"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6817","nonce":468601409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6819","nonce":468601409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6823","nonce":468601409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6821","nonce":468601409}]},"public_addr":"172.21.15.167:6817/468601409","cluster_addr":"172.21.15.167:6819/468601409","heartbeat_back_addr":"172.21.15.167:6823/468601409","heartbeat_front_addr":"172.21.15.167:6821/468601409","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.433172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:31.159802+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.624825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:43.147 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2024-08-31T08:19:43.147 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2024-08-31T08:19:43.431 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:43.431 INFO:teuthology.orchestra.run.smithi167.stdout:{"epoch":16,"fsid":"c9c7e778-1922-4472-806b-f799b2b74be3","created":"2024-08-31T08:19:23.593759+0000","modified":"2024-08-31T08:19:42.177055+0000","last_up_change":"2024-08-31T08:19:33.677518+0000","last_in_change":"2024-08-31T08:19:24.429956+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-31T08:19:35.151358+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-31T08:19:38.645516+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"4272991b-2bd9-4b65-a7fc-c4f6e5d05c75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6800","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6801","nonce":3684510860}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6802","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6803","nonce":3684510860}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6806","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6807","nonce":3684510860}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6804","nonce":3684510860},{"type":"v1","addr":"172.21.15.167:6805","nonce":3684510860}]},"public_addr":"172.21.15.167:6801/3684510860","cluster_addr":"172.21.15.167:6803/3684510860","heartbeat_back_addr":"172.21.15.167:6807/3684510860","heartbeat_front_addr":"172.21.15.167:6805/3684510860","state":["exists","up"]},{"osd":1,"uuid":"1b93f235-4e86-4c42-8601-b05b5b777cac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6808","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6809","nonce":1387293444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6810","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6811","nonce":1387293444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6814","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6815","nonce":1387293444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6812","nonce":1387293444},{"type":"v1","addr":"172.21.15.167:6813","nonce":1387293444}]},"public_addr":"172.21.15.167:6809/1387293444","cluster_addr":"172.21.15.167:6811/1387293444","heartbeat_back_addr":"172.21.15.167:6815/1387293444","heartbeat_front_addr":"172.21.15.167:6813/1387293444","state":["exists","up"]},{"osd":2,"uuid":"92b6bc52-ef47-41b3-8c2e-c7ff34518eb7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6816","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6817","nonce":468601409}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6818","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6819","nonce":468601409}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6822","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6823","nonce":468601409}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.167:6820","nonce":468601409},{"type":"v1","addr":"172.21.15.167:6821","nonce":468601409}]},"public_addr":"172.21.15.167:6817/468601409","cluster_addr":"172.21.15.167:6819/468601409","heartbeat_back_addr":"172.21.15.167:6823/468601409","heartbeat_front_addr":"172.21.15.167:6821/468601409","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.433172+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:31.159802+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-08-31T08:19:30.624825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-31T08:19:43.444 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2024-08-31T08:19:43.445 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2024-08-31T08:19:43.445 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2024-08-31T08:19:43.583 INFO:teuthology.orchestra.run.smithi167.stdout:38654705669 2024-08-31T08:19:43.584 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-08-31T08:19:43.585 INFO:teuthology.orchestra.run.smithi167.stdout:38654705669 2024-08-31T08:19:43.585 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-08-31T08:19:43.586 INFO:teuthology.orchestra.run.smithi167.stdout:38654705670 2024-08-31T08:19:43.586 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-08-31T08:19:43.924 INFO:teuthology.orchestra.run.smithi167.stdout:38654705669 2024-08-31T08:19:43.930 INFO:teuthology.orchestra.run.smithi167.stdout:38654705668 2024-08-31T08:19:43.932 INFO:teuthology.orchestra.run.smithi167.stdout:38654705668 2024-08-31T08:19:43.938 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705669 for osd.0 2024-08-31T08:19:43.944 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.2 2024-08-31T08:19:43.946 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.1 2024-08-31T08:19:44.940 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2024-08-31T08:19:44.945 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2024-08-31T08:19:44.947 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2024-08-31T08:19:45.250 INFO:teuthology.orchestra.run.smithi167.stdout:38654705671 2024-08-31T08:19:45.263 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705670 got 38654705671 for osd.0 2024-08-31T08:19:45.263 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:45.289 INFO:teuthology.orchestra.run.smithi167.stdout:38654705670 2024-08-31T08:19:45.296 INFO:teuthology.orchestra.run.smithi167.stdout:38654705670 2024-08-31T08:19:45.303 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705670 for osd.1 2024-08-31T08:19:45.303 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:45.309 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705670 for osd.2 2024-08-31T08:19:45.309 DEBUG:teuthology.parallel:result is None 2024-08-31T08:19:45.309 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2024-08-31T08:19:45.309 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-08-31T08:19:45.629 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:45.629 INFO:teuthology.orchestra.run.smithi167.stderr:dumped all 2024-08-31T08:19:45.641 INFO:teuthology.orchestra.run.smithi167.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-08-31T08:19:45.136812+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":86836,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":85179,"kb_avail":283028684,"statfs":{"total":289910292480,"available":289821372416,"internally_reserved":0,"allocated":1613824,"data_stored":1302373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":87223631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.965445"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184853+0000","last_change":"2024-08-31T08:19:42.184853+0000","last_active":"2024-08-31T08:19:42.184853+0000","last_peered":"2024-08-31T08:19:42.184853+0000","last_clean":"2024-08-31T08:19:42.184853+0000","last_became_active":"2024-08-31T08:19:40.178682+0000","last_became_peered":"2024-08-31T08:19:40.178682+0000","last_unstale":"2024-08-31T08:19:42.184853+0000","last_undegraded":"2024-08-31T08:19:42.184853+0000","last_fullsized":"2024-08-31T08:19:42.184853+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T10:17:53.108086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067018799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184689+0000","last_change":"2024-08-31T08:19:42.184689+0000","last_active":"2024-08-31T08:19:42.184689+0000","last_peered":"2024-08-31T08:19:42.184689+0000","last_clean":"2024-08-31T08:19:42.184689+0000","last_became_active":"2024-08-31T08:19:40.178591+0000","last_became_peered":"2024-08-31T08:19:40.178591+0000","last_unstale":"2024-08-31T08:19:42.184689+0000","last_undegraded":"2024-08-31T08:19:42.184689+0000","last_fullsized":"2024-08-31T08:19:42.184689+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T15:41:52.118670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055068999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.185157+0000","last_change":"2024-08-31T08:19:42.185157+0000","last_active":"2024-08-31T08:19:42.185157+0000","last_peered":"2024-08-31T08:19:42.185157+0000","last_clean":"2024-08-31T08:19:42.185157+0000","last_became_active":"2024-08-31T08:19:40.178559+0000","last_became_peered":"2024-08-31T08:19:40.178559+0000","last_unstale":"2024-08-31T08:19:42.185157+0000","last_undegraded":"2024-08-31T08:19:42.185157+0000","last_fullsized":"2024-08-31T08:19:42.185157+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T20:00:08.528267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089267600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184992+0000","last_change":"2024-08-31T08:19:42.184992+0000","last_active":"2024-08-31T08:19:42.184992+0000","last_peered":"2024-08-31T08:19:42.184992+0000","last_clean":"2024-08-31T08:19:42.184992+0000","last_became_active":"2024-08-31T08:19:40.178624+0000","last_became_peered":"2024-08-31T08:19:40.178624+0000","last_unstale":"2024-08-31T08:19:42.184992+0000","last_undegraded":"2024-08-31T08:19:42.184992+0000","last_fullsized":"2024-08-31T08:19:42.184992+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T11:43:18.779261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073274899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.183476+0000","last_change":"2024-08-31T08:19:42.181922+0000","last_active":"2024-08-31T08:19:42.183476+0000","last_peered":"2024-08-31T08:19:42.183476+0000","last_clean":"2024-08-31T08:19:42.183476+0000","last_became_active":"2024-08-31T08:19:40.178948+0000","last_became_peered":"2024-08-31T08:19:40.178948+0000","last_unstale":"2024-08-31T08:19:42.183476+0000","last_undegraded":"2024-08-31T08:19:42.183476+0000","last_fullsized":"2024-08-31T08:19:42.183476+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T14:42:25.703070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047893399999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.341768+0000","last_change":"2024-08-31T08:19:42.341768+0000","last_active":"2024-08-31T08:19:42.341768+0000","last_peered":"2024-08-31T08:19:42.341768+0000","last_clean":"2024-08-31T08:19:42.341768+0000","last_became_active":"2024-08-31T08:19:40.180592+0000","last_became_peered":"2024-08-31T08:19:40.180592+0000","last_unstale":"2024-08-31T08:19:42.341768+0000","last_undegraded":"2024-08-31T08:19:42.341768+0000","last_fullsized":"2024-08-31T08:19:42.341768+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:33:53.980048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034447699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.341808+0000","last_change":"2024-08-31T08:19:42.341808+0000","last_active":"2024-08-31T08:19:42.341808+0000","last_peered":"2024-08-31T08:19:42.341808+0000","last_clean":"2024-08-31T08:19:42.341808+0000","last_became_active":"2024-08-31T08:19:40.179311+0000","last_became_peered":"2024-08-31T08:19:40.179311+0000","last_unstale":"2024-08-31T08:19:42.341808+0000","last_undegraded":"2024-08-31T08:19:42.341808+0000","last_fullsized":"2024-08-31T08:19:42.341808+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T18:07:34.686925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040163699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184575+0000","last_change":"2024-08-31T08:19:42.184575+0000","last_active":"2024-08-31T08:19:42.184575+0000","last_peered":"2024-08-31T08:19:42.184575+0000","last_clean":"2024-08-31T08:19:42.184575+0000","last_became_active":"2024-08-31T08:19:40.179126+0000","last_became_peered":"2024-08-31T08:19:40.179126+0000","last_unstale":"2024-08-31T08:19:42.184575+0000","last_undegraded":"2024-08-31T08:19:42.184575+0000","last_fullsized":"2024-08-31T08:19:42.184575+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T18:23:55.149803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052185700000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":87,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.183899+0000","last_change":"2024-08-31T08:19:36.168108+0000","last_active":"2024-08-31T08:19:42.183899+0000","last_peered":"2024-08-31T08:19:42.183899+0000","last_clean":"2024-08-31T08:19:42.183899+0000","last_became_active":"2024-08-31T08:19:36.167083+0000","last_became_peered":"2024-08-31T08:19:36.167083+0000","last_unstale":"2024-08-31T08:19:42.183899+0000","last_undegraded":"2024-08-31T08:19:42.183899+0000","last_fullsized":"2024-08-31T08:19:42.183899+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:35.151803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:35.151803+0000","last_clean_scrub_stamp":"2024-08-31T08:19:35.151803+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T10:30:06.324478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":139264,"data_stored":40533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31912,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339928,"statfs":{"total":96636764160,"available":96604086272,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T08:19:45.642 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2024-08-31T08:19:45.920 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:45.920 INFO:teuthology.orchestra.run.smithi167.stderr:dumped all 2024-08-31T08:19:45.932 INFO:teuthology.orchestra.run.smithi167.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-08-31T08:19:45.136812+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":81,"num_write_kb":1108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":86836,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":85179,"kb_avail":283028684,"statfs":{"total":289910292480,"available":289821372416,"internally_reserved":0,"allocated":1613824,"data_stored":1302373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4785,"internal_metadata":87223631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.965445"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184853+0000","last_change":"2024-08-31T08:19:42.184853+0000","last_active":"2024-08-31T08:19:42.184853+0000","last_peered":"2024-08-31T08:19:42.184853+0000","last_clean":"2024-08-31T08:19:42.184853+0000","last_became_active":"2024-08-31T08:19:40.178682+0000","last_became_peered":"2024-08-31T08:19:40.178682+0000","last_unstale":"2024-08-31T08:19:42.184853+0000","last_undegraded":"2024-08-31T08:19:42.184853+0000","last_fullsized":"2024-08-31T08:19:42.184853+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T10:17:53.108086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067018799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184689+0000","last_change":"2024-08-31T08:19:42.184689+0000","last_active":"2024-08-31T08:19:42.184689+0000","last_peered":"2024-08-31T08:19:42.184689+0000","last_clean":"2024-08-31T08:19:42.184689+0000","last_became_active":"2024-08-31T08:19:40.178591+0000","last_became_peered":"2024-08-31T08:19:40.178591+0000","last_unstale":"2024-08-31T08:19:42.184689+0000","last_undegraded":"2024-08-31T08:19:42.184689+0000","last_fullsized":"2024-08-31T08:19:42.184689+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T15:41:52.118670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055068999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.185157+0000","last_change":"2024-08-31T08:19:42.185157+0000","last_active":"2024-08-31T08:19:42.185157+0000","last_peered":"2024-08-31T08:19:42.185157+0000","last_clean":"2024-08-31T08:19:42.185157+0000","last_became_active":"2024-08-31T08:19:40.178559+0000","last_became_peered":"2024-08-31T08:19:40.178559+0000","last_unstale":"2024-08-31T08:19:42.185157+0000","last_undegraded":"2024-08-31T08:19:42.185157+0000","last_fullsized":"2024-08-31T08:19:42.185157+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T20:00:08.528267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089267600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184992+0000","last_change":"2024-08-31T08:19:42.184992+0000","last_active":"2024-08-31T08:19:42.184992+0000","last_peered":"2024-08-31T08:19:42.184992+0000","last_clean":"2024-08-31T08:19:42.184992+0000","last_became_active":"2024-08-31T08:19:40.178624+0000","last_became_peered":"2024-08-31T08:19:40.178624+0000","last_unstale":"2024-08-31T08:19:42.184992+0000","last_undegraded":"2024-08-31T08:19:42.184992+0000","last_fullsized":"2024-08-31T08:19:42.184992+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T11:43:18.779261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073274899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.183476+0000","last_change":"2024-08-31T08:19:42.181922+0000","last_active":"2024-08-31T08:19:42.183476+0000","last_peered":"2024-08-31T08:19:42.183476+0000","last_clean":"2024-08-31T08:19:42.183476+0000","last_became_active":"2024-08-31T08:19:40.178948+0000","last_became_peered":"2024-08-31T08:19:40.178948+0000","last_unstale":"2024-08-31T08:19:42.183476+0000","last_undegraded":"2024-08-31T08:19:42.183476+0000","last_fullsized":"2024-08-31T08:19:42.183476+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T14:42:25.703070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047893399999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.341768+0000","last_change":"2024-08-31T08:19:42.341768+0000","last_active":"2024-08-31T08:19:42.341768+0000","last_peered":"2024-08-31T08:19:42.341768+0000","last_clean":"2024-08-31T08:19:42.341768+0000","last_became_active":"2024-08-31T08:19:40.180592+0000","last_became_peered":"2024-08-31T08:19:40.180592+0000","last_unstale":"2024-08-31T08:19:42.341768+0000","last_undegraded":"2024-08-31T08:19:42.341768+0000","last_fullsized":"2024-08-31T08:19:42.341768+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T13:33:53.980048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034447699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.341808+0000","last_change":"2024-08-31T08:19:42.341808+0000","last_active":"2024-08-31T08:19:42.341808+0000","last_peered":"2024-08-31T08:19:42.341808+0000","last_clean":"2024-08-31T08:19:42.341808+0000","last_became_active":"2024-08-31T08:19:40.179311+0000","last_became_peered":"2024-08-31T08:19:40.179311+0000","last_unstale":"2024-08-31T08:19:42.341808+0000","last_undegraded":"2024-08-31T08:19:42.341808+0000","last_fullsized":"2024-08-31T08:19:42.341808+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T18:07:34.686925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040163699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.184575+0000","last_change":"2024-08-31T08:19:42.184575+0000","last_active":"2024-08-31T08:19:42.184575+0000","last_peered":"2024-08-31T08:19:42.184575+0000","last_clean":"2024-08-31T08:19:42.184575+0000","last_became_active":"2024-08-31T08:19:40.179126+0000","last_became_peered":"2024-08-31T08:19:40.179126+0000","last_unstale":"2024-08-31T08:19:42.184575+0000","last_undegraded":"2024-08-31T08:19:42.184575+0000","last_fullsized":"2024-08-31T08:19:42.184575+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:39.168549+0000","last_clean_scrub_stamp":"2024-08-31T08:19:39.168549+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T18:23:55.149803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052185700000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'52","reported_seq":87,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-08-31T08:19:42.183899+0000","last_change":"2024-08-31T08:19:36.168108+0000","last_active":"2024-08-31T08:19:42.183899+0000","last_peered":"2024-08-31T08:19:42.183899+0000","last_clean":"2024-08-31T08:19:42.183899+0000","last_became_active":"2024-08-31T08:19:36.167083+0000","last_became_peered":"2024-08-31T08:19:36.167083+0000","last_unstale":"2024-08-31T08:19:42.183899+0000","last_undegraded":"2024-08-31T08:19:42.183899+0000","last_fullsized":"2024-08-31T08:19:42.183899+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-31T08:19:35.151803+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-31T08:19:35.151803+0000","last_clean_scrub_stamp":"2024-08-31T08:19:35.151803+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-01T10:30:06.324478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":56,"num_read_kb":46,"num_write":79,"num_write_kb":1106,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52,"ondisk_log_size":52,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705670,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":139264,"data_stored":40533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705670,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31912,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":31166,"kb_avail":94339928,"statfs":{"total":96636764160,"available":96604086272,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":31914437},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705671,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27756,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344084,"statfs":{"total":96636764160,"available":96608342016,"internally_reserved":0,"allocated":737280,"data_stored":630920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-31T08:19:45.933 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2024-08-31T08:19:45.933 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2024-08-31T08:19:45.933 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2024-08-31T08:19:45.933 DEBUG:teuthology.orchestra.run.smithi167:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2024-08-31T08:19:46.251 INFO:teuthology.orchestra.run.smithi167.stdout: 2024-08-31T08:19:46.251 INFO:teuthology.orchestra.run.smithi167.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-31T08:19:46.264 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2024-08-31T08:19:46.264 INFO:teuthology.run_tasks:Running task exec... 2024-08-31T08:19:46.276 INFO:teuthology.task.exec:Executing custom commands... 2024-08-31T08:19:46.276 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@smithi167.front.sepia.ceph.com 2024-08-31T08:19:46.277 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:46.637 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror] 2024-08-31T08:19:46.658 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQDW0dJmrw/6KhAACJu5QZpXVnZm0wv0EgS4Dw== 2024-08-31T08:19:46.658 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror-peer" 2024-08-31T08:19:46.658 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:46.659 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror 2024-08-31T08:19:46.659 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:47.023 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.0] 2024-08-31T08:19:47.023 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQDW0dJm4QlLLhAANebu3lwKrCTMMpbQVurUcg== 2024-08-31T08:19:47.023 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:47.023 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:47.023 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.0 2024-08-31T08:19:47.036 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:47.398 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.1] 2024-08-31T08:19:47.398 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQDW0dJmGLDhMxAAB7bY6pr+8sc1+iXsi2f3dQ== 2024-08-31T08:19:47.398 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:47.398 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:47.398 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.1 2024-08-31T08:19:47.412 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:47.772 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.2] 2024-08-31T08:19:47.772 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQDW0dJmL7p5ORAAEZkhhom3ksV2CIBBK01zng== 2024-08-31T08:19:47.772 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:47.773 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:47.773 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.2 2024-08-31T08:19:47.786 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:48.146 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.3] 2024-08-31T08:19:48.146 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQDX0dJmIbp8AxAA+2aYROwihhlV773Ig8N1Mg== 2024-08-31T08:19:48.146 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:48.146 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:48.146 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.3 2024-08-31T08:19:48.159 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:48.527 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror] 2024-08-31T08:19:48.527 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmcZZJCxAAwN8G66HmpmipVSPYEDR3dg== 2024-08-31T08:19:48.527 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror-peer" 2024-08-31T08:19:48.527 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:48.527 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror 2024-08-31T08:19:48.541 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:48.906 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.0] 2024-08-31T08:19:48.906 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmx+nlEBAAP/sOoja7Kcr6uPO5mXgV5w== 2024-08-31T08:19:48.906 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:48.907 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:48.907 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.0 2024-08-31T08:19:48.920 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:49.284 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.1] 2024-08-31T08:19:49.284 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJm/8N4FhAAs8RBI8IDZ8jGj60BOlSnsw== 2024-08-31T08:19:49.284 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:49.285 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:49.285 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.1 2024-08-31T08:19:49.298 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:49.664 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.2] 2024-08-31T08:19:49.664 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmH98UHBAAkCOzZ36QMkfR8VWjMZMxZA== 2024-08-31T08:19:49.664 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:49.664 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:49.664 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.2 2024-08-31T08:19:49.678 DEBUG:teuthology.orchestra.run.smithi167:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2024-08-31T08:19:50.041 INFO:teuthology.orchestra.run.smithi167.stdout:[client.mirror.3] 2024-08-31T08:19:50.041 INFO:teuthology.orchestra.run.smithi167.stdout: key = AQD90dJmPvWoIRAA7e9aQ4SpPdzYLSRpAdN5kw== 2024-08-31T08:19:50.041 INFO:teuthology.orchestra.run.smithi167.stdout: caps mon = "profile rbd-mirror" 2024-08-31T08:19:50.041 INFO:teuthology.orchestra.run.smithi167.stdout: caps osd = "profile rbd" 2024-08-31T08:19:50.042 INFO:teuthology.orchestra.run.smithi167.stderr:updated caps for client.mirror.3 2024-08-31T08:19:50.056 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:19:50.073 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:19:50.075 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:19:50.075 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.084 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:19:50.084 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:19:50.086 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:19:50.086 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.094 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-08-31T08:19:50.094 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-08-31T08:19:50.097 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-08-31T08:19:50.097 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.106 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-08-31T08:19:50.106 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-08-31T08:19:50.108 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-08-31T08:19:50.108 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-08-31T08:19:50.116 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-08-31T08:19:50.119 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-08-31T08:19:50.119 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.127 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:19:50.127 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:19:50.130 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:19:50.130 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-08-31T08:19:50.140 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-08-31T08:19:50.143 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-08-31T08:19:50.143 INFO:teuthology.run_tasks:Running task rbd-mirror... 2024-08-31T08:19:50.151 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-08-31T08:19:50.152 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-08-31T08:19:50.155 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-08-31T08:19:50.155 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-08-31T08:19:50.167 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1725092390 2024-08-31T08:19:50.167 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-08-31T08:19:50.167 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2024-08-31T08:19:50.177 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1725092390 2024-08-31T08:19:50.177 DEBUG:tasks.rbd_mirror_thrash:Yielding 2024-08-31T08:19:50.177 INFO:teuthology.run_tasks:Running task workunit... 2024-08-31T08:19:50.198 INFO:tasks.workunit:Pulling workunits from ref dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:19:50.198 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-31T08:19:50.198 DEBUG:teuthology.orchestra.run.smithi167:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-08-31T08:19:50.199 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2024-08-31T08:19:50.199 INFO:tasks.rbd_mirror_thrash:waiting for 86 secs before thrashing 2024-08-31T08:19:50.200 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2024-08-31T08:19:50.200 INFO:tasks.rbd_mirror_thrash:waiting for 76 secs before thrashing 2024-08-31T08:19:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-31T08:19:50.218 INFO:teuthology.orchestra.run.smithi167.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2024-08-31T08:19:50.218 DEBUG:teuthology.orchestra.run.smithi167:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-08-31T08:19:50.275 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2024-08-31T08:19:50.275 DEBUG:teuthology.orchestra.run.smithi167:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2024-08-31T08:19:50.330 INFO:tasks.workunit:timeout=6h 2024-08-31T08:19:50.331 INFO:tasks.workunit:cleanup=True 2024-08-31T08:19:50.331 DEBUG:teuthology.orchestra.run.smithi167:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout dc9f1ccbfcab74acda17f39d6ab018266071e760 2024-08-31T08:19:50.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2024-08-31T08:20:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Note: switching to 'dc9f1ccbfcab74acda17f39d6ab018266071e760'. 2024-08-31T08:20:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:changes and commit them, and you can discard any commits you make in this 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:state without impacting any branches by switching back to a branch. 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:If you want to create a new branch to retain commits you create, you may 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:do so (now or later) by using -c with the switch command. Example: 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: git switch -c 2024-08-31T08:20:39.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Or undo this operation with: 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: git switch - 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:20:39.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:HEAD is now at dc9f1ccbfca Merge branch 'wip-nbalacha-async-sorted-snaps' of https://github.com/nbalacha/ceph into HEAD 2024-08-31T08:20:39.969 DEBUG:teuthology.orchestra.run.smithi167:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2024-08-31T08:20:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-31T08:20:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-08-31T08:20:40.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-31T08:20:40.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-31T08:20:40.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-31T08:20:40.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2024-08-31T08:20:40.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-08-31T08:20:40.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-31T08:20:40.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2024-08-31T08:20:40.840 DEBUG:teuthology.orchestra.run.smithi167:> set -ex 2024-08-31T08:20:40.840 DEBUG:teuthology.orchestra.run.smithi167:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2024-08-31T08:20:40.896 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2024-08-31T08:20:40.897 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2024-08-31T08:20:40.897 DEBUG:teuthology.orchestra.run.smithi167:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=dc9f1ccbfcab74acda17f39d6ab018266071e760 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-08-31T08:20:40.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ IMAGE_COUNT=50 2024-08-31T08:20:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ export LOCKDEP=0 2024-08-31T08:20:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ LOCKDEP=0 2024-08-31T08:20:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-08-31T08:20:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2024-08-31T08:20:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ RBD_MIRROR_INSTANCES=4 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CLUSTER1=cluster1 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CLUSTER2=cluster2 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ PEER_CLUSTER_SUFFIX= 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ POOL=mirror 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ PARENT_POOL=mirror_parent 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ NS1=ns1 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ NS2=ns2 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ TEMPDIR= 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ID=mirror 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ RBD_MIRROR_MODE=snapshot 2024-08-31T08:20:40.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ MIRROR_POOL_MODE=image 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' snapshot = snapshot ']' 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ MIRROR_POOL_MODE=image 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ export 'CEPH_ARGS=--id mirror' 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS='--id mirror' 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ LAST_MIRROR_INSTANCE=3 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2024-08-31T08:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2024-08-31T08:20:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_BIN=. 2024-08-31T08:20:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_SRC=. 2024-08-31T08:20:40.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' -e CMakeCache.txt ']' 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 -gt 0 ']' 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ setup 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local c 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ trap 'cleanup $?' INT TERM EXIT 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ setup_tempdir 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:40.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mktemp -d 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ TEMPDIR=/tmp/tmp.V6fpnqxu3N 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -z 1 ']' 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ setup_pools cluster1 cluster2 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster2 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mon_map_file 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mon_addr 2024-08-31T08:20:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local admin_key_file 2024-08-31T08:20:40.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid 2024-08-31T08:20:40.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:40.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2024-08-31T08:20:41.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:pool 'mirror' already exists 2024-08-31T08:20:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:41.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2024-08-31T08:20:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:pool 'mirror_parent' already exists 2024-08-31T08:20:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 pool init mirror 2024-08-31T08:20:45.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:45.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2024-08-31T08:20:48.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:48.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:48.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2024-08-31T08:20:48.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2024-08-31T08:20:48.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2024-08-31T08:20:48.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2024-08-31T08:20:48.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2024-08-31T08:20:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2024-08-31T08:20:48.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -z ']' 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -z ']' 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_add cluster1 mirror cluster2 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local client_cluster=cluster2 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster2 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid_var_name 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:48.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local error_code 2024-08-31T08:20:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local peer_uuid 2024-08-31T08:20:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 16 32 2024-08-31T08:20:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:20:48.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2024-08-31T08:20:48.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_uuid=e82eadfc-e2b2-418a-998f-413f4a86c94d 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=0 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 -eq 17 ']' 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 0 -eq 0 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_add cluster1 mirror_parent cluster2 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror_parent 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local client_cluster=cluster2 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:48.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster2 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid_var_name 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local error_code 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local peer_uuid 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 16 32 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:20:48.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_uuid=39b7862d-c6c7-4e80-9773-0eeb435984a8 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=0 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 -eq 17 ']' 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 0 -eq 0 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ setup_pools cluster2 cluster1 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster1 2024-08-31T08:20:48.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mon_map_file 2024-08-31T08:20:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mon_addr 2024-08-31T08:20:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local admin_key_file 2024-08-31T08:20:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid 2024-08-31T08:20:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2024-08-31T08:20:49.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:pool 'mirror' already exists 2024-08-31T08:20:49.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:49.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2024-08-31T08:20:50.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:20:50.279+0000 7fd8ffb44640 -1 rbd::mirror::PoolReplayer: 0x561a4f850c00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:50.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:20:50.280+0000 7eff3229f640 -1 rbd::mirror::PoolReplayer: 0x55ce4fb92c00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:50.285 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi167.stderr:2024-08-31T08:20:50.281+0000 7fdf16215640 -1 rbd::mirror::PoolReplayer: 0x5560a4e5ec00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:50.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:20:50.280+0000 7f376951e640 -1 rbd::mirror::PoolReplayer: 0x5602088f4c00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:50.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:pool 'mirror_parent' already exists 2024-08-31T08:20:50.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:50.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 pool init mirror 2024-08-31T08:20:51.278 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:20:51.276+0000 7fd8ffb44640 -1 rbd::mirror::PoolReplayer: 0x561a4fa60f00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:51.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:20:51.276+0000 7f376951e640 -1 rbd::mirror::PoolReplayer: 0x560208a02f00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:51.279 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi167.stderr:2024-08-31T08:20:51.277+0000 7fdf16215640 -1 rbd::mirror::PoolReplayer: 0x5560a5062f00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:51.280 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:20:51.278+0000 7eff3229f640 -1 rbd::mirror::PoolReplayer: 0x55ce4fc96f00 init: failed to initialize remote pool poller: (2) No such file or directory 2024-08-31T08:20:53.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:53.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2024-08-31T08:20:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ CEPH_ARGS= 2024-08-31T08:20:56.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2024-08-31T08:20:56.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2024-08-31T08:20:56.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2024-08-31T08:20:56.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2024-08-31T08:20:56.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2024-08-31T08:20:56.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2024-08-31T08:20:56.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -z ']' 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -z ']' 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_add cluster2 mirror cluster1 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local client_cluster=cluster1 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster1 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid_var_name 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local error_code 2024-08-31T08:20:56.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local peer_uuid 2024-08-31T08:20:56.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 16 32 2024-08-31T08:20:56.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:20:56.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_uuid=ece1a143-dbf3-4c45-af07-d0c203bd422b 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=0 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 -eq 17 ']' 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 0 -eq 0 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:20:56.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_add cluster2 mirror_parent cluster1 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror_parent 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local client_cluster=cluster1 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local remote_cluster=cluster1 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local uuid_var_name 2024-08-31T08:20:56.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local error_code 2024-08-31T08:20:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local peer_uuid 2024-08-31T08:20:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 16 32 2024-08-31T08:20:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:20:56.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ peer_uuid=cdfa022b-c50e-4581-b810-8f4be080a519 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=0 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 -eq 17 ']' 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 0 -eq 0 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirrors cluster1 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 0 3 2024-08-31T08:20:56.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster1:0 2024-08-31T08:20:56.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1:0 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster1:0 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster1 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=0 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 = cluster1:0 ']' 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster1 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=0 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:20:56.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster1:1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1:1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster1:1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=1 2024-08-31T08:20:56.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 1 = cluster1:1 ']' 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster1 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=1 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster1:2 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1:2 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster1:2 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster1 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=2 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 2 = cluster1:2 ']' 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster1 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=2 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster1:3 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1:3 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster1:3 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster1 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=3 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 3 = cluster1:3 ']' 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster1 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=3 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirrors cluster2 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 0 3 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster2:0 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2:0 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster2:0 cluster instance 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster2:0 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster2 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=0 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 0 = cluster2:0 ']' 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster2 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster2 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=0 2024-08-31T08:20:56.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster2:1 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2:1 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster2:1 cluster instance 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster2:1 2024-08-31T08:20:56.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster2 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=1 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 1 = cluster2:1 ']' 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster2 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster2 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=1 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster2:2 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2:2 2024-08-31T08:20:56.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster2:2 cluster instance 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster2:2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 2 = cluster2:2 ']' 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster2:3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2:3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster2:3 cluster instance 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster2:3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster2 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 3 = cluster2:3 ']' 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster2 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster2 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:20:56.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:20:56 TEST: add image and test replay after client crashes 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:20:56 TEST: add image and test replay after client crashes 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=test 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:20:56.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror test 512M 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:20:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror test snapshot 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:20:56.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2024-08-31T08:20:57.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:20:57.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/test 2024-08-31T08:20:57.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:20:57.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10b33a338723 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10b33a338723 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:20:56 2024 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:20:56 2024 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:20:56 2024 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:20:57.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:20:57.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:57.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:57.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:57.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:20:57.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:20:57.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:20:57.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:20:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:20:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:20:58.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:20:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:20:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:20:58.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:58.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:20:58.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:20:58.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:20:58.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:58.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:20:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:58.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:20:58.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:20:58.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:20:58.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:20:58.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:58.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:58.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:20:58.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:58.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:58.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:20:58.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:20:58.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:58.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:58.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:20:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:58.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:58.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:58.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:20:58.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:20:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:20:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:20:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:20:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:20:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:58.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:58.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:20:58.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 1 2024-08-31T08:20:59.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:20:59.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:59.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:59.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:20:59.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:20:59.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:20:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:20:59.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:20:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:59.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:59.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:20:59.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:20:59.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:20:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:20:59.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:20:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:20:59.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:20:59.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:20:59.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:20:59.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:20:59.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:20:59.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:20:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:20:59.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:21:00.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:00.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:00.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:21:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:21:00.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:21:00.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:21:00.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:00.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:00.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 2 2024-08-31T08:21:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:21:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:21:02.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:02.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:21:02.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:21:02.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:02.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:21:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:21:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:21:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:21:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:02.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:02.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:02.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:21:02.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:21:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:02.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:21:02.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:02.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:02.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:02.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:21:02.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:21:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:21:02.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:02.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:02.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:02.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:21:02.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:02.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:02.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:02.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:02.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:21:02.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:21:02.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:21:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:02.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:02.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:02.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 4 2024-08-31T08:21:06.201 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5154109479011496) > weight (0.25) 2024-08-31T08:21:06.201 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:21:06.202 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:21:06.202 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:21:06.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:21:06.202+0000 7f2d90f49640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 27016) UID: 1000 2024-08-31T08:21:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:21:06.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:21:06.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:06.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:21:06.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:21:06.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:06.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:06.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:21:06.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:21:06.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:06.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:06.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:06.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:21:06.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:06.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:21:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:21:06.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:06.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:06.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:21:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:21:06.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:21:06.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:06.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:06.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:06.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:21:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:06.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:21:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:21:06.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:21:06.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:06.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:06.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:21:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:06.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:06.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:06.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:21:06.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:21:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:06.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:21:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:06.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:06.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:21:06.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:21:06.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:21:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:06.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:21:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:06.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:06.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 8 2024-08-31T08:21:11.203 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:21:11.204 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:21:11.204 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:21:11.204 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:21:11.204 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:21:11.207 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:21:11.207 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-08-31T08:21:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:21:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:14.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:21:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:21:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:21:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:14.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:21:15.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:15.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:15.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:15.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:21:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:21:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:21:15.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:15.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:15.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:21:15.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:21:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:21:15.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:21:15.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:15.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:21:15.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:21:15.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:21:15.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:15.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:15.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:21:15.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:21:15.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:15.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:15.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:21:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:21:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:15.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:21:15.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:21:15.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:21:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:21:15.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:15.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:15.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:21:15.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:21:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:21:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:21:15.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:21:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:15.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 8 2024-08-31T08:21:16.200 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6530821657105004) > weight (0.25) 2024-08-31T08:21:16.201 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-08-31T08:21:16.201 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-08-31T08:21:16.201 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:21:16.202 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:21:16.201+0000 7eff2f242640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 26948) UID: 1000 2024-08-31T08:21:20.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:21:20.286+0000 7f376951e640 -1 rbd::mirror::Mirror: 0x5602087fe000 update_pool_replayers: restarting failed pool replayer for uuid: e82eadfc-e2b2-418a-998f-413f4a86c94d cluster: cluster2 client: client.mirror 2024-08-31T08:21:20.287 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:21:20.286+0000 7fd8ffb44640 -1 rbd::mirror::Mirror: 0x561a4f87c000 update_pool_replayers: restarting failed pool replayer for uuid: e82eadfc-e2b2-418a-998f-413f4a86c94d cluster: cluster2 client: client.mirror 2024-08-31T08:21:20.288 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi167.stderr:2024-08-31T08:21:20.286+0000 7fdf16215640 -1 rbd::mirror::Mirror: 0x5560a41fc000 update_pool_replayers: restarting failed pool replayer for uuid: e82eadfc-e2b2-418a-998f-413f4a86c94d cluster: cluster2 client: client.mirror 2024-08-31T08:21:20.364 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:21:20.363+0000 7fd8ffb44640 -1 rbd::mirror::Mirror: 0x561a4f87c000 update_pool_replayers: restarting failed pool replayer for uuid: 39b7862d-c6c7-4e80-9773-0eeb435984a8 cluster: cluster2 client: client.mirror 2024-08-31T08:21:20.396 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi167.stderr:2024-08-31T08:21:20.395+0000 7fdf16215640 -1 rbd::mirror::Mirror: 0x5560a41fc000 update_pool_replayers: restarting failed pool replayer for uuid: 39b7862d-c6c7-4e80-9773-0eeb435984a8 cluster: cluster2 client: client.mirror 2024-08-31T08:21:20.412 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:21:20.411+0000 7f376951e640 -1 rbd::mirror::Mirror: 0x5602087fe000 update_pool_replayers: restarting failed pool replayer for uuid: 39b7862d-c6c7-4e80-9773-0eeb435984a8 cluster: cluster2 client: client.mirror 2024-08-31T08:21:23.202 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-08-31T08:21:23.203 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:21:23.203 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-08-31T08:21:23.203 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:21:23.203 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:21:23.206 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:21:23.207 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-08-31T08:21:23.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:21:23.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:21:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:21:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:21:23.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:21:23.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:21:23.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:21:23.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:21:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:21:23.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:21:23.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:21:23.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:21:23.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:21:23.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:21:23.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name= 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 10 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:21:23.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:21:23.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:21:23.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:23.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:23.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:21:23.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:21:23.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:21:23.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:21:23.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 37962752 bytes 2024-08-31T08:21:23.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 70706176 bytes 2024-08-31T08:21:23.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 79900672 bytes 2024-08-31T08:21:23.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 115099648 bytes 2024-08-31T08:21:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 196569088 bytes 2024-08-31T08:21:24.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 267260928 bytes 2024-08-31T08:21:24.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 276240384 bytes 2024-08-31T08:21:24.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 304793600 bytes 2024-08-31T08:21:24.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 394319872 bytes 2024-08-31T08:21:24.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 446143488 bytes 2024-08-31T08:21:25.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 475717632 bytes 2024-08-31T08:21:25.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 501999616 bytes 2024-08-31T08:21:25.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 515489792 bytes 2024-08-31T08:21:25.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 542022656 bytes 2024-08-31T08:21:25.518 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 584970240 bytes 2024-08-31T08:21:25.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 606834688 bytes 2024-08-31T08:21:25.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 662144000 bytes 2024-08-31T08:21:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 683116544 bytes 2024-08-31T08:21:26.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 723964928 bytes 2024-08-31T08:21:26.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 745292800 bytes 2024-08-31T08:21:26.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 772339712 bytes 2024-08-31T08:21:26.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 797220864 bytes 2024-08-31T08:21:26.433 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 822945792 bytes 2024-08-31T08:21:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 888379392 bytes 2024-08-31T08:21:26.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 977932288 bytes 2024-08-31T08:21:26.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 984333312 bytes 2024-08-31T08:21:27.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1067467776 bytes 2024-08-31T08:21:28.347 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1148957696 bytes 2024-08-31T08:21:28.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1175821312 bytes 2024-08-31T08:21:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1235796992 bytes 2024-08-31T08:21:29.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1238975488 bytes 2024-08-31T08:21:29.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1249711104 bytes 2024-08-31T08:21:29.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1306255360 bytes 2024-08-31T08:21:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1329458176 bytes 2024-08-31T08:21:29.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1353146368 bytes 2024-08-31T08:21:30.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1433172992 bytes 2024-08-31T08:21:30.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1499681792 bytes 2024-08-31T08:21:30.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1518203904 bytes 2024-08-31T08:21:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1573170176 bytes 2024-08-31T08:21:30.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1607179264 bytes 2024-08-31T08:21:30.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1681762304 bytes 2024-08-31T08:21:31.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1760680960 bytes 2024-08-31T08:21:31.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1799461888 bytes 2024-08-31T08:21:31.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1878299648 bytes 2024-08-31T08:21:31.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1972241408 bytes 2024-08-31T08:21:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1979267072 bytes 2024-08-31T08:21:32.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2016772096 bytes 2024-08-31T08:21:32.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2108515328 bytes 2024-08-31T08:21:32.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2130058240 bytes 2024-08-31T08:21:32.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2204320768 bytes 2024-08-31T08:21:32.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2242359296 bytes 2024-08-31T08:21:32.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2325227520 bytes 2024-08-31T08:21:33.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2424164352 bytes 2024-08-31T08:21:33.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2454748160 bytes 2024-08-31T08:21:33.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2516712448 bytes 2024-08-31T08:21:33.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2545762304 bytes 2024-08-31T08:21:33.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2577000448 bytes 2024-08-31T08:21:33.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2632214528 bytes 2024-08-31T08:21:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2663335936 bytes 2024-08-31T08:21:34.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2665151488 bytes 2024-08-31T08:21:34.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2710203392 bytes 2024-08-31T08:21:34.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2743447552 bytes 2024-08-31T08:21:34.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2751407104 bytes 2024-08-31T08:21:34.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2755640320 bytes 2024-08-31T08:21:34.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2770807808 bytes 2024-08-31T08:21:34.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2827030528 bytes 2024-08-31T08:21:34.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2871687168 bytes 2024-08-31T08:21:35.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2918243328 bytes 2024-08-31T08:21:35.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2952633344 bytes 2024-08-31T08:21:35.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3000760320 bytes 2024-08-31T08:21:35.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3097624576 bytes 2024-08-31T08:21:35.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3122592768 bytes 2024-08-31T08:21:35.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3195199488 bytes 2024-08-31T08:21:35.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3196837888 bytes 2024-08-31T08:21:36.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3203417088 bytes 2024-08-31T08:21:36.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3219982336 bytes 2024-08-31T08:21:36.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3254534144 bytes 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:36.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:36.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:21:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:21:37.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:21:37.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+unknown 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: status not found 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31T08:21:37.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:21:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:21:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+unknown 2024-08-31T08:21:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: status not found 2024-08-31T08:21:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: ' 2024-08-31T08:21:37.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:21:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 1 2024-08-31T08:21:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:37.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 2 2024-08-31T08:21:39.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:21:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:39.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:21:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:21:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:21:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:21:39.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:21:39.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:21:39.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+unknown 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: status not found 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+unknown 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: status not found 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: ' 2024-08-31T08:21:39.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:21:39.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 1 2024-08-31T08:21:39.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:39.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 4 2024-08-31T08:21:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:21:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:43.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:43.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:21:43.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:21:43.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:21:43.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:21:43.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:21:43.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:21:43.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+unknown 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: status not found 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:21:41' 2024-08-31T08:21:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:21:43.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 1 2024-08-31T08:21:43.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:21:43.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 8 2024-08-31T08:21:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:21:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:51.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:21:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:21:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:21:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:21:51.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:21:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:21:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092457,"remote_snapshot_timestamp":1725092457,"replay_state":"idle"} 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:21:50 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:21:41' 2024-08-31T08:21:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:21:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:21:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:21:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092457,"remote_snapshot_timestamp":1725092457,"replay_state":"idle"} 2024-08-31T08:21:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:21:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:21:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:21:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:21:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:21:51.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:21:51.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:21:51.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:21:51.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:21:50' 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:21:50' 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:21:50' 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:21:51.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:21:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42214300mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43294332mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092457,"remote_snapshot_timestamp":1725092457,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:21:50cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:21:41' 2024-08-31T08:21:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:21:51.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:21:51.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:21:50' 2024-08-31T08:21:51.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:21:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:21:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:21:51.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:21:51.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:21:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:21:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:21:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap1 2024-08-31T08:21:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap1 2024-08-31T08:21:51.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:21:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:21:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:21:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1 2024-08-31T08:21:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:22:05.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:22:05.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:22:06.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:22:06.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:22:06.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:22:06.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:22:06.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:06.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:22:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:22:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:22:06.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:22:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:22:06.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:22:06.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:22:06.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:06.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:22:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:06.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:22:07.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 5 2024-08-31T08:22:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:07.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:22:07.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:07.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:22:07.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=5 2024-08-31T08:22:07.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:07.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:07.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:07.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=3 2024-08-31T08:22:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 5 = 3 2024-08-31T08:22:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:08.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:08.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:08.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:08.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:08.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 5 = '' 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:08.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:22:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:08.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:08.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:08.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:08.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:08.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:08.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:22:08.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:22:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 5 = '' 2024-08-31T08:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:08.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:22:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:10.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:10.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:10.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=5 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 5 = 5 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:22:10.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:22:11.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2024-08-31T08:22:11.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:22:11.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:22:11.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:22:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:22:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:22:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:22:11.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:22:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap1 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:22:11.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:22:12.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:22:12.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:22:13.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:22:13.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:22:13.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap1-clean 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap1-clean 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1-clean 2024-08-31T08:22:13.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:22:14.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:22:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:22:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:14.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:22:14.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:22:14.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:22:14.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:22:14.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 43520000 bytes 2024-08-31T08:22:14.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 108904448 bytes 2024-08-31T08:22:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 124059648 bytes 2024-08-31T08:22:15.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 181556224 bytes 2024-08-31T08:22:15.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 203567104 bytes 2024-08-31T08:22:15.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 265684992 bytes 2024-08-31T08:22:15.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 321763328 bytes 2024-08-31T08:22:15.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 364265472 bytes 2024-08-31T08:22:15.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 384282624 bytes 2024-08-31T08:22:15.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 434957312 bytes 2024-08-31T08:22:16.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 461196288 bytes 2024-08-31T08:22:16.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 564454400 bytes 2024-08-31T08:22:16.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 613657600 bytes 2024-08-31T08:22:16.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 643643392 bytes 2024-08-31T08:22:16.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 677604352 bytes 2024-08-31T08:22:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 715577344 bytes 2024-08-31T08:22:17.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 760181760 bytes 2024-08-31T08:22:17.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 790902784 bytes 2024-08-31T08:22:17.278 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 820386816 bytes 2024-08-31T08:22:17.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 883211264 bytes 2024-08-31T08:22:17.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 914439168 bytes 2024-08-31T08:22:17.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1046686720 bytes 2024-08-31T08:22:18.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1069717504 bytes 2024-08-31T08:22:18.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1088320512 bytes 2024-08-31T08:22:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1116436480 bytes 2024-08-31T08:22:18.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1163094016 bytes 2024-08-31T08:22:18.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1207609344 bytes 2024-08-31T08:22:18.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1219528704 bytes 2024-08-31T08:22:18.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1264522240 bytes 2024-08-31T08:22:18.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1320876032 bytes 2024-08-31T08:22:19.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1337191424 bytes 2024-08-31T08:22:19.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1372480512 bytes 2024-08-31T08:22:19.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1417900032 bytes 2024-08-31T08:22:19.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1441843200 bytes 2024-08-31T08:22:19.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1477936128 bytes 2024-08-31T08:22:19.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1585392640 bytes 2024-08-31T08:22:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1615837184 bytes 2024-08-31T08:22:20.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1678641152 bytes 2024-08-31T08:22:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1697323008 bytes 2024-08-31T08:22:20.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1747203072 bytes 2024-08-31T08:22:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1824320512 bytes 2024-08-31T08:22:20.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1848739840 bytes 2024-08-31T08:22:20.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1851631616 bytes 2024-08-31T08:22:21.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1899167744 bytes 2024-08-31T08:22:21.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1958721536 bytes 2024-08-31T08:22:21.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2012145664 bytes 2024-08-31T08:22:21.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2029113344 bytes 2024-08-31T08:22:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2089989120 bytes 2024-08-31T08:22:21.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2090585088 bytes 2024-08-31T08:22:21.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2147079168 bytes 2024-08-31T08:22:22.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2203680768 bytes 2024-08-31T08:22:22.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2244745216 bytes 2024-08-31T08:22:22.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2246051840 bytes 2024-08-31T08:22:22.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2261106688 bytes 2024-08-31T08:22:22.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2312572928 bytes 2024-08-31T08:22:22.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2384924672 bytes 2024-08-31T08:22:22.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2483670016 bytes 2024-08-31T08:22:23.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2544593920 bytes 2024-08-31T08:22:23.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2555200512 bytes 2024-08-31T08:22:23.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2605039616 bytes 2024-08-31T08:22:23.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2610748416 bytes 2024-08-31T08:22:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2674880512 bytes 2024-08-31T08:22:23.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2738442240 bytes 2024-08-31T08:22:23.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2742349824 bytes 2024-08-31T08:22:24.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2826296320 bytes 2024-08-31T08:22:24.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2889524224 bytes 2024-08-31T08:22:24.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2936378368 bytes 2024-08-31T08:22:24.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2993477632 bytes 2024-08-31T08:22:24.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2999908352 bytes 2024-08-31T08:22:24.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3025455104 bytes 2024-08-31T08:22:25.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3144903680 bytes 2024-08-31T08:22:25.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3212476416 bytes 2024-08-31T08:22:25.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3270133760 bytes 2024-08-31T08:22:25.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3296222208 bytes 2024-08-31T08:22:25.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3364827136 bytes 2024-08-31T08:22:25.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3379928064 bytes 2024-08-31T08:22:26.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3431429120 bytes 2024-08-31T08:22:26.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3467379712 bytes 2024-08-31T08:22:26.208 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4771363455974217) > weight (0.25) 2024-08-31T08:22:26.208 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:22:26.209 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:22:26.209 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:22:26.209 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:22:26.209+0000 7f821be92640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 28287) UID: 1000 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:26.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:22:26.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:22:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:22:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:27.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:22:27.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:22:27.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:22:27.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:22:27.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:22:27.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:22:27.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:22:27.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":17706291.2,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"} 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:22:20 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:22:27.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:22:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: down+stopped 2024-08-31T08:22:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:22:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:22:11' 2024-08-31T08:22:27.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:22:27.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:22:27.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:22:27.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":17706291.2,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"} 2024-08-31T08:22:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:22:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:22:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:22:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:22:27.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:22:27.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:22:20' 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:22:20' 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:22:20' 2024-08-31T08:22:27.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:22:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:22:27.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:22:27.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='WARNINGOKWARNING142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42214300mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43294332mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":17706291.2,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:22:20cluster2b2ed8de6-469c-4293-9d02-39498e569bf2down+stoppedlocal image is primary2024-08-31 08:22:11' 2024-08-31T08:22:27.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:22:27.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:22:27.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:22:20' 2024-08-31T08:22:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:22:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:22:27.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:22:27.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:22:27.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:22:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap2 2024-08-31T08:22:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap2 2024-08-31T08:22:27.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2 2024-08-31T08:22:27.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:22:31.210 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:22:31.210 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:22:31.210 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:22:31.210 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:22:31.211 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:22:31.213 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:22:31.214 INFO:tasks.rbd_mirror_thrash:waiting for 74 secs before thrashing 2024-08-31T08:22:38.208 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7876537033167579) > weight (0.25) 2024-08-31T08:22:38.208 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5051742453392619) > weight (0.25) 2024-08-31T08:22:38.208 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-08-31T08:22:38.209 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-08-31T08:22:38.209 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:22:38.210 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:22:38.209+0000 7f37664c1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 26966) UID: 1000 2024-08-31T08:22:45.210 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-08-31T08:22:45.210 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:22:45.211 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-08-31T08:22:45.211 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-08-31T08:22:45.211 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-08-31T08:22:45.214 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-08-31T08:22:45.214 INFO:tasks.rbd_mirror_thrash:waiting for 93 secs before thrashing 2024-08-31T08:22:55.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:22:55.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:22:56.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:22:56.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:22:56.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:22:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:22:56.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:22:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:22:56.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:22:56.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:22:56.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:22:56.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:22:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:56.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:22:56.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:22:56.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:22:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:22:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:22:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:22:56.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:22:56.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:22:56.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:22:56.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:22:56.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:22:56.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:56.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:22:56.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:56.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:22:58.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 8 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:22:58.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:22:58.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:58.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:22:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=8 2024-08-31T08:22:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:58.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:22:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:22:58.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:22:58.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:22:58.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 8 = '' 2024-08-31T08:22:58.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:58.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:22:59.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:22:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:22:59.246 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:22:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:22:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:22:59.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:22:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:22:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 8 = '' 2024-08-31T08:22:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:22:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:00.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:00.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:23:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:00.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:23:00.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:23:00.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:23:00.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:23:00.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 8 = '' 2024-08-31T08:23:00.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:23:00.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:01.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:01.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:01.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=8 2024-08-31T08:23:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 8 = 8 2024-08-31T08:23:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2024-08-31T08:23:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:23:01.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:23:02.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2024-08-31T08:23:02.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 4 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:23:02.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:23:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:23:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:23:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:23:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:23:02.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:23:02.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:23:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:02.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap1-clean ']' 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap1-clean 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1-clean 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:23:02.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:02.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:02.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:02.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:03.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:03.433 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:04.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:04.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:04.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap2 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:23:04.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:04.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:05.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:05.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:05.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:05.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:06.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap2-clean 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap2-clean 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:06.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2-clean 2024-08-31T08:23:06.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:06.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:23:06.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:23:06.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:23:06.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:23:06.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 36005888 bytes 2024-08-31T08:23:07.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 111293440 bytes 2024-08-31T08:23:07.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 129144832 bytes 2024-08-31T08:23:07.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 150621184 bytes 2024-08-31T08:23:07.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 204968960 bytes 2024-08-31T08:23:07.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 236720128 bytes 2024-08-31T08:23:07.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 288794624 bytes 2024-08-31T08:23:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 363575296 bytes 2024-08-31T08:23:08.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 456284160 bytes 2024-08-31T08:23:08.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 499136512 bytes 2024-08-31T08:23:08.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 645916672 bytes 2024-08-31T08:23:08.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 686212096 bytes 2024-08-31T08:23:09.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 743977984 bytes 2024-08-31T08:23:09.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 808783872 bytes 2024-08-31T08:23:09.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 826196992 bytes 2024-08-31T08:23:09.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 877523968 bytes 2024-08-31T08:23:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 884530176 bytes 2024-08-31T08:23:09.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 905364480 bytes 2024-08-31T08:23:09.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 947739648 bytes 2024-08-31T08:23:10.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 983631872 bytes 2024-08-31T08:23:10.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1042065408 bytes 2024-08-31T08:23:10.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1116290048 bytes 2024-08-31T08:23:10.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1165704192 bytes 2024-08-31T08:23:10.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1212867584 bytes 2024-08-31T08:23:10.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1243570176 bytes 2024-08-31T08:23:11.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1323669504 bytes 2024-08-31T08:23:11.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1371394048 bytes 2024-08-31T08:23:11.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1371998208 bytes 2024-08-31T08:23:11.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1439758336 bytes 2024-08-31T08:23:11.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1494891520 bytes 2024-08-31T08:23:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1504782336 bytes 2024-08-31T08:23:11.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1532907520 bytes 2024-08-31T08:23:12.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1549581312 bytes 2024-08-31T08:23:12.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1596276736 bytes 2024-08-31T08:23:12.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1633038336 bytes 2024-08-31T08:23:12.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1638582272 bytes 2024-08-31T08:23:12.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1684156416 bytes 2024-08-31T08:23:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1716805632 bytes 2024-08-31T08:23:12.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1726344192 bytes 2024-08-31T08:23:12.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1766990848 bytes 2024-08-31T08:23:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1825329152 bytes 2024-08-31T08:23:13.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1831856128 bytes 2024-08-31T08:23:13.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1854680064 bytes 2024-08-31T08:23:13.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1899326464 bytes 2024-08-31T08:23:13.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1963118592 bytes 2024-08-31T08:23:13.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2006610944 bytes 2024-08-31T08:23:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2091672576 bytes 2024-08-31T08:23:14.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2106539008 bytes 2024-08-31T08:23:14.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2123207680 bytes 2024-08-31T08:23:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2217658368 bytes 2024-08-31T08:23:14.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2273832960 bytes 2024-08-31T08:23:14.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2357728256 bytes 2024-08-31T08:23:15.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2448056320 bytes 2024-08-31T08:23:15.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2482618368 bytes 2024-08-31T08:23:15.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2522196992 bytes 2024-08-31T08:23:15.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2629937152 bytes 2024-08-31T08:23:15.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2670968832 bytes 2024-08-31T08:23:16.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2695556096 bytes 2024-08-31T08:23:16.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2734660608 bytes 2024-08-31T08:23:16.278 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2761437184 bytes 2024-08-31T08:23:16.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2769516544 bytes 2024-08-31T08:23:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2824466432 bytes 2024-08-31T08:23:16.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2892119040 bytes 2024-08-31T08:23:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2959140864 bytes 2024-08-31T08:23:17.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 3019777024 bytes 2024-08-31T08:23:17.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 3046594560 bytes 2024-08-31T08:23:17.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 3069382656 bytes 2024-08-31T08:23:17.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 3151860736 bytes 2024-08-31T08:23:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:23:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:23:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:23:17.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:23:17.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:23:17.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:23:17.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:23:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:23:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:23:18.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:23:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:23:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:23:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:23:18.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"} 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:22:50 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:22:51' 2024-08-31T08:23:18.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:23:18.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:23:18.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:23:18.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"} 2024-08-31T08:23:18.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:23:18.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:23:18.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:23:18.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:23:18.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:23:18.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:23:18.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:23:18.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:22:50' 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:22:50' 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:22:50' 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:23:18.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:23:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43294332mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":265594368.0,"last_snapshot_bytes":531188736,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092527,"remote_snapshot_timestamp":1725092527,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:22:50cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:22:51' 2024-08-31T08:23:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:23:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:23:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:22:50' 2024-08-31T08:23:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:23:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:23:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:23:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:23:18.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap3 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap3 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:18.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:18.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3 2024-08-31T08:23:18.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:23:45.215 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8354732795935007) > weight (0.25) 2024-08-31T08:23:45.215 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.38975908195571296) > weight (0.25) 2024-08-31T08:23:45.215 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8766324784611416) > weight (0.25) 2024-08-31T08:23:45.215 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6431182951007729) > weight (0.25) 2024-08-31T08:23:45.215 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:48.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:23:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:23:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:23:48.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:23:49.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:23:49.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:23:49.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:23:49.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:23:49.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:23:49.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:23:49.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:23:49.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:23:49.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:23:49.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:23:49.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:23:49.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:23:49.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:23:49.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:23:49.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:23:49.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:23:49.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:23:49.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:23:49.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:23:49.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:23:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:23:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:23:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:23:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:23:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:23:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:23:49.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:49.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:23:51.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 11 2024-08-31T08:23:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:23:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:51.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:23:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:23:51.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:51.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:23:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=11 2024-08-31T08:23:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:23:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:23:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:23:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:51.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:23:51.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:23:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:23:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:23:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:23:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 11 = '' 2024-08-31T08:23:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:23:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:23:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:52.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:52.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:52.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:23:52.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:23:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:23:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:23:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:23:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 11 = '' 2024-08-31T08:23:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:23:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:23:52.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:52.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:52.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:52.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:52.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:53.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:23:53.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:23:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:23:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:23:53.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:23:53.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 11 = '' 2024-08-31T08:23:53.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:23:53.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:23:53.601 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:23:53.599+0000 7f9dc3f75640 -1 reset not still connected to 0x55a09b4f2b60 2024-08-31T08:23:53.601 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:23:53.600+0000 7f9dc3f75640 -1 reset not still connected to 0x55a09be6a750 2024-08-31T08:23:53.601 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:23:53.600+0000 7f9dc3f75640 -1 reset not still connected to 0x55a09c68e750 2024-08-31T08:23:53.601 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:23:53.600+0000 7f9dc3f75640 -1 reset not still connected to 0x55a09c68ef70 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:23:54.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:23:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:23:54.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:23:54.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=11 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 11 = 11 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:54.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3 2024-08-31T08:23:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:23:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:23:54.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:23:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 6 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:23:55.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:23:03 2024 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:23:55.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap2-clean ']' 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap2-clean 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:55.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2-clean 2024-08-31T08:23:55.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:23:55.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:55.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:55.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:55.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:56.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:56.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:57.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:57.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:57.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap3 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3 2024-08-31T08:23:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:23:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:57.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:23:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:58.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:58.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:23:58.746 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:58.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:23:59.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap3-clean 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap3-clean 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3-clean 2024-08-31T08:23:59.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:23:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:23:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:23:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:23:59.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:23:59.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:23:59.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:23:59.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:23:59.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 58121216 bytes 2024-08-31T08:24:00.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 143620096 bytes 2024-08-31T08:24:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 189326336 bytes 2024-08-31T08:24:00.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 218016768 bytes 2024-08-31T08:24:00.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 297032704 bytes 2024-08-31T08:24:00.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 383030272 bytes 2024-08-31T08:24:01.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 427698176 bytes 2024-08-31T08:24:01.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 475206656 bytes 2024-08-31T08:24:01.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 499544064 bytes 2024-08-31T08:24:01.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 508360704 bytes 2024-08-31T08:24:01.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 560851968 bytes 2024-08-31T08:24:01.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 604354560 bytes 2024-08-31T08:24:01.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 663131136 bytes 2024-08-31T08:24:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 668284928 bytes 2024-08-31T08:24:02.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 732992512 bytes 2024-08-31T08:24:02.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 766040064 bytes 2024-08-31T08:24:02.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 767054848 bytes 2024-08-31T08:24:02.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 827211776 bytes 2024-08-31T08:24:02.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 903775232 bytes 2024-08-31T08:24:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 935623680 bytes 2024-08-31T08:24:03.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 995587072 bytes 2024-08-31T08:24:03.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 996202496 bytes 2024-08-31T08:24:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1058735104 bytes 2024-08-31T08:24:03.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1107635200 bytes 2024-08-31T08:24:03.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1184436224 bytes 2024-08-31T08:24:03.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1218744320 bytes 2024-08-31T08:24:04.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1262685184 bytes 2024-08-31T08:24:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1286037504 bytes 2024-08-31T08:24:04.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1296735232 bytes 2024-08-31T08:24:04.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1361387520 bytes 2024-08-31T08:24:04.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1385400320 bytes 2024-08-31T08:24:04.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1426332672 bytes 2024-08-31T08:24:04.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1443303424 bytes 2024-08-31T08:24:04.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1471727616 bytes 2024-08-31T08:24:05.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1519017984 bytes 2024-08-31T08:24:05.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1557775360 bytes 2024-08-31T08:24:05.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1574725632 bytes 2024-08-31T08:24:05.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1610329088 bytes 2024-08-31T08:24:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1628727296 bytes 2024-08-31T08:24:05.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1666240512 bytes 2024-08-31T08:24:05.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1715697664 bytes 2024-08-31T08:24:05.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1749030912 bytes 2024-08-31T08:24:06.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1766909952 bytes 2024-08-31T08:24:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1792421888 bytes 2024-08-31T08:24:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1827782656 bytes 2024-08-31T08:24:06.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1861218304 bytes 2024-08-31T08:24:06.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1868652544 bytes 2024-08-31T08:24:06.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1875064832 bytes 2024-08-31T08:24:06.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1923987456 bytes 2024-08-31T08:24:06.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1958145024 bytes 2024-08-31T08:24:07.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2011739136 bytes 2024-08-31T08:24:07.273 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2069783552 bytes 2024-08-31T08:24:07.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2173315072 bytes 2024-08-31T08:24:07.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2228346880 bytes 2024-08-31T08:24:07.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2251983872 bytes 2024-08-31T08:24:07.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2260567040 bytes 2024-08-31T08:24:08.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2270600192 bytes 2024-08-31T08:24:08.163 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2311287808 bytes 2024-08-31T08:24:08.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2379188224 bytes 2024-08-31T08:24:08.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2423945216 bytes 2024-08-31T08:24:08.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2470794240 bytes 2024-08-31T08:24:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2533704704 bytes 2024-08-31T08:24:09.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2621744128 bytes 2024-08-31T08:24:09.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2680958976 bytes 2024-08-31T08:24:09.432 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2756644864 bytes 2024-08-31T08:24:09.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2763831296 bytes 2024-08-31T08:24:09.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2782836736 bytes 2024-08-31T08:24:09.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2842685440 bytes 2024-08-31T08:24:09.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2862352384 bytes 2024-08-31T08:24:10.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2910609408 bytes 2024-08-31T08:24:10.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2966252544 bytes 2024-08-31T08:24:10.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3007091712 bytes 2024-08-31T08:24:10.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3037019136 bytes 2024-08-31T08:24:10.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3060321280 bytes 2024-08-31T08:24:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3099130880 bytes 2024-08-31T08:24:10.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3140623360 bytes 2024-08-31T08:24:11.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3158635520 bytes 2024-08-31T08:24:11.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3193029632 bytes 2024-08-31T08:24:11.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3217814528 bytes 2024-08-31T08:24:11.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3272094720 bytes 2024-08-31T08:24:11.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3278839808 bytes 2024-08-31T08:24:11.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8200 IOs, 3291508736 bytes 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:11.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:24:11.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:24:11.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:24:11.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:24:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:24:12.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:24:12.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:24:12.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533113856.0,"last_snapshot_bytes":535038976,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092577,"remote_snapshot_timestamp":1725092577,"replay_state":"idle"} 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:23:50 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:23:51' 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:24:12.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:24:12.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:24:12.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533113856.0,"last_snapshot_bytes":535038976,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092577,"remote_snapshot_timestamp":1725092577,"replay_state":"idle"} 2024-08-31T08:24:12.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:24:12.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:24:12.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:24:12.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:24:12.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:24:12.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:24:12.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:24:12.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:24:12.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:23:50' 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:23:50' 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:23:50' 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:24:12.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:24:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43294332mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":533113856.0,"last_snapshot_bytes":535038976,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092577,"remote_snapshot_timestamp":1725092577,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:23:50cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:23:51' 2024-08-31T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:23:50' 2024-08-31T08:24:12.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:24:12.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:24:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:24:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:24:12.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:24:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:12.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap4 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap4 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4 2024-08-31T08:24:12.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:24:18.215 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6496344710583163) > weight (0.25) 2024-08-31T08:24:18.216 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-08-31T08:24:18.216 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-08-31T08:24:18.217 INFO:tasks.rbd_mirror_thrash:waiting for 0 secs before reviving daemons 2024-08-31T08:24:18.217 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-08-31T08:24:18.217 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:24:18.217 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:24:18.216+0000 7f26643c5640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 28815) UID: 1000 2024-08-31T08:24:24.318 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-08-31T08:24:24.318 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:24:24.318 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:24:24.322 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:24:24.322 INFO:tasks.rbd_mirror_thrash:waiting for 116 secs before thrashing 2024-08-31T08:24:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:24:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:41.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:24:41.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:24:42.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:24:42.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:42.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:24:42.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:24:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:24:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:24:42.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:24:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:24:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:24:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:24:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:24:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:24:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:24:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:24:42.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:24:42.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:24:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:24:42.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:24:42.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:24:42.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:24:42.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:24:42.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:24:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:24:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:24:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:24:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:24:42.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:24:42.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:42.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:24:42.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:42.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:42.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:42.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:24:44.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 14 2024-08-31T08:24:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:24:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:44.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:24:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:24:44.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:24:44.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:24:44.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=14 2024-08-31T08:24:44.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:24:44.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:24:44.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:24:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:44.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:44.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:44.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:44.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:24:44.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:24:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:24:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:24:44.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:24:44.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:24:44.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:24:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:24:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 14 = '' 2024-08-31T08:24:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:24:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:24:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:45.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:24:45.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:24:45.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:24:45.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:24:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:24:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:24:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:24:45.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:24:45.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 14 = '' 2024-08-31T08:24:45.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:24:45.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:24:45.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:24:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:24:45.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:24:45.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:24:45.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:24:45.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:24:45.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:24:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 14 = '' 2024-08-31T08:24:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:24:45.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:24:47.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:24:47.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:24:47.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=14 2024-08-31T08:24:47.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 14 = 14 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:24:47.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:24:48.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 8 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:24:48.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:23:03 2024 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:24:48.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap3-clean ']' 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap3-clean 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3-clean 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:48.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:48.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:24:49.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:24:49.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:50.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:24:50.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:50.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap4 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:24:50.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:50.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:50.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:24:51.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:24:51.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:51.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:24:51.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:51.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:24:52.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap4-clean 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap4-clean 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4-clean 2024-08-31T08:24:52.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:52.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:24:52.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:24:52.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:24:52.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:24:52.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 24235008 bytes 2024-08-31T08:24:52.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 40856576 bytes 2024-08-31T08:24:52.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 126152704 bytes 2024-08-31T08:24:53.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 161082368 bytes 2024-08-31T08:24:53.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 161686528 bytes 2024-08-31T08:24:53.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 190255104 bytes 2024-08-31T08:24:53.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 197056512 bytes 2024-08-31T08:24:53.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 239845376 bytes 2024-08-31T08:24:53.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 277356544 bytes 2024-08-31T08:24:53.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 351436800 bytes 2024-08-31T08:24:53.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 384787456 bytes 2024-08-31T08:24:54.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 433972224 bytes 2024-08-31T08:24:54.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 498387968 bytes 2024-08-31T08:24:54.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 614588416 bytes 2024-08-31T08:24:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 639406080 bytes 2024-08-31T08:24:54.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 697091072 bytes 2024-08-31T08:24:55.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 741035008 bytes 2024-08-31T08:24:55.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 790947840 bytes 2024-08-31T08:24:55.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 805346304 bytes 2024-08-31T08:24:55.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 915788800 bytes 2024-08-31T08:24:55.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 972537856 bytes 2024-08-31T08:24:55.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1029933056 bytes 2024-08-31T08:24:56.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1116226560 bytes 2024-08-31T08:24:56.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1128517632 bytes 2024-08-31T08:24:56.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1164235776 bytes 2024-08-31T08:24:56.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1243928576 bytes 2024-08-31T08:24:56.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1309740032 bytes 2024-08-31T08:24:57.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1361505280 bytes 2024-08-31T08:24:57.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1372211200 bytes 2024-08-31T08:24:57.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1428748288 bytes 2024-08-31T08:24:57.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1472651264 bytes 2024-08-31T08:24:57.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1516351488 bytes 2024-08-31T08:24:57.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1527620608 bytes 2024-08-31T08:24:57.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1549149184 bytes 2024-08-31T08:24:57.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1562918912 bytes 2024-08-31T08:24:58.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1583137792 bytes 2024-08-31T08:24:58.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1583724544 bytes 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:58.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:24:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:24:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:24:58.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:24:59.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:24:59.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:24:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:24:59.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":15239304.53,"bytes_per_snapshot":535326208.0,"last_snapshot_bytes":536665088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092683,"remote_snapshot_timestamp":1725092683,"replay_state":"idle"} 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:24:50 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:24:51' 2024-08-31T08:24:59.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:24:59.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:24:59.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:24:59.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":15239304.53,"bytes_per_snapshot":535326208.0,"last_snapshot_bytes":536665088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092683,"remote_snapshot_timestamp":1725092683,"replay_state":"idle"} 2024-08-31T08:24:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:24:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:24:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:24:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:24:59.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:24:59.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:24:59.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:24:59.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:24:59.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:24:50' 2024-08-31T08:24:59.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:24:50' 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:24:50' 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:24:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:24:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":15239304.53,"bytes_per_snapshot":535326208.0,"last_snapshot_bytes":536665088,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092683,"remote_snapshot_timestamp":1725092683,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:24:50cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:24:51' 2024-08-31T08:24:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:24:59.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:24:59.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:24:50' 2024-08-31T08:24:59.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:24:59.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:24:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:24:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:24:59.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:24:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:24:59.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap5 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap5 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5 2024-08-31T08:24:59.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:25:00.217 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-08-31T08:25:00.217 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-08-31T08:25:00.217 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-08-31T08:25:00.218 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:25:00.218+0000 7f8f1ede9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 27000) UID: 1000 2024-08-31T08:25:02.219 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-08-31T08:25:02.219 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:25:02.220 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-08-31T08:25:02.220 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-08-31T08:25:02.220 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-08-31T08:25:02.223 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-08-31T08:25:02.223 INFO:tasks.rbd_mirror_thrash:waiting for 78 secs before thrashing 2024-08-31T08:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:25:30.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:25:30.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:25:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:25:30.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:25:30.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:25:30.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:25:30.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:25:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:25:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:25:30.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:25:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:25:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:25:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:25:30.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:25:30.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:25:30.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:25:30.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:25:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:25:32.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 17 2024-08-31T08:25:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:25:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:32.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:25:32.347 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:25:32.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:25:32.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:25:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=17 2024-08-31T08:25:32.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:25:32.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:25:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:32.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:25:32.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:25:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:25:32.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:25:32.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:25:32.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:25:32.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:25:32.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:25:32.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 17 = '' 2024-08-31T08:25:32.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:25:32.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:25:33.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:25:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:25:33.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:25:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:25:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:25:33.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:25:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:25:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 17 = '' 2024-08-31T08:25:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:25:33.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:25:33.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:25:33.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:25:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:25:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:25:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:25:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:25:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:25:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:25:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 17 = '' 2024-08-31T08:25:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:25:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:25:35.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:25:35.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:25:35.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:25:35.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=17 2024-08-31T08:25:35.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 17 = 17 2024-08-31T08:25:35.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:35.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2024-08-31T08:25:35.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:25:35.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:25:36.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2024-08-31T08:25:36.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:25:36.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:25:36.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:25:36.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 10 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:24:49 2024 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:25:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:25:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:25:36.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap4-clean ']' 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap4-clean 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4-clean 2024-08-31T08:25:36.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:25:36.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:25:36.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:36.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:36.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:25:37.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:25:37.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:38.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:25:38.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:38.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap5 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:38.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:38.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:25:38.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:25:38.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:39.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:25:39.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:39.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap5-clean 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap5-clean 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5-clean 2024-08-31T08:25:39.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:25:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:25:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:25:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:40.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:40.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:25:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:25:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:25:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:25:40.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 67459072 bytes 2024-08-31T08:25:41.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 117691392 bytes 2024-08-31T08:25:41.250 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 118319104 bytes 2024-08-31T08:25:41.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 118938624 bytes 2024-08-31T08:25:41.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 171570176 bytes 2024-08-31T08:25:41.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 176558080 bytes 2024-08-31T08:25:41.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 199783424 bytes 2024-08-31T08:25:41.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 241366016 bytes 2024-08-31T08:25:41.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 286369792 bytes 2024-08-31T08:25:42.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 358597632 bytes 2024-08-31T08:25:42.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 382062592 bytes 2024-08-31T08:25:42.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 382674944 bytes 2024-08-31T08:25:42.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 429156352 bytes 2024-08-31T08:25:42.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 478899200 bytes 2024-08-31T08:25:42.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 498127872 bytes 2024-08-31T08:25:42.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 523693056 bytes 2024-08-31T08:25:43.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 562940928 bytes 2024-08-31T08:25:43.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 608791552 bytes 2024-08-31T08:25:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 672477184 bytes 2024-08-31T08:25:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 731324416 bytes 2024-08-31T08:25:43.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 746118144 bytes 2024-08-31T08:25:43.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 755014656 bytes 2024-08-31T08:25:44.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 849372160 bytes 2024-08-31T08:25:44.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 912035840 bytes 2024-08-31T08:25:44.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 972229632 bytes 2024-08-31T08:25:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1053481984 bytes 2024-08-31T08:25:44.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1065614336 bytes 2024-08-31T08:25:44.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1115894784 bytes 2024-08-31T08:25:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1141442560 bytes 2024-08-31T08:25:45.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1242087424 bytes 2024-08-31T08:25:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1279813632 bytes 2024-08-31T08:25:45.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1316111360 bytes 2024-08-31T08:25:45.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1325320192 bytes 2024-08-31T08:25:45.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1353157632 bytes 2024-08-31T08:25:45.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1362256896 bytes 2024-08-31T08:25:45.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1377871872 bytes 2024-08-31T08:25:46.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1574876160 bytes 2024-08-31T08:25:46.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1639718912 bytes 2024-08-31T08:25:46.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1668881408 bytes 2024-08-31T08:25:46.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1699234816 bytes 2024-08-31T08:25:47.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1761082368 bytes 2024-08-31T08:25:47.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1851439104 bytes 2024-08-31T08:25:47.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1872467968 bytes 2024-08-31T08:25:47.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1930226688 bytes 2024-08-31T08:25:47.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1953177600 bytes 2024-08-31T08:25:47.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1977324544 bytes 2024-08-31T08:25:48.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2047257600 bytes 2024-08-31T08:25:48.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2107848704 bytes 2024-08-31T08:25:48.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2168673280 bytes 2024-08-31T08:25:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2218511360 bytes 2024-08-31T08:25:48.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2248519680 bytes 2024-08-31T08:25:48.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2297560064 bytes 2024-08-31T08:25:48.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2320546816 bytes 2024-08-31T08:25:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2423985152 bytes 2024-08-31T08:25:49.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2442127360 bytes 2024-08-31T08:25:49.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2469718016 bytes 2024-08-31T08:25:49.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2535161856 bytes 2024-08-31T08:25:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2587083776 bytes 2024-08-31T08:25:50.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2634231808 bytes 2024-08-31T08:25:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2642045952 bytes 2024-08-31T08:25:50.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2667203584 bytes 2024-08-31T08:25:50.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2703405056 bytes 2024-08-31T08:25:50.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2734523392 bytes 2024-08-31T08:25:50.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2775165952 bytes 2024-08-31T08:25:50.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2839579648 bytes 2024-08-31T08:25:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2840383488 bytes 2024-08-31T08:25:51.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2929449984 bytes 2024-08-31T08:25:51.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2948014080 bytes 2024-08-31T08:25:51.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2960157696 bytes 2024-08-31T08:25:51.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3004302336 bytes 2024-08-31T08:25:51.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3047316480 bytes 2024-08-31T08:25:51.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3160068096 bytes 2024-08-31T08:25:52.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3201181696 bytes 2024-08-31T08:25:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3236543488 bytes 2024-08-31T08:25:52.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3312203776 bytes 2024-08-31T08:25:52.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3324484608 bytes 2024-08-31T08:25:52.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:25:52.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:25:52.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:25:52.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:25:52.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:25:53.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:25:53.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":16741580.8,"bytes_per_snapshot":519456256.0,"last_snapshot_bytes":502247424,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"} 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.0 on smithi167 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:25:50 2024-08-31T08:25:53.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:25:51' 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:25:53.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:25:53.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":16741580.8,"bytes_per_snapshot":519456256.0,"last_snapshot_bytes":502247424,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"} 2024-08-31T08:25:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:25:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:25:53.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:25:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:25:53.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:25:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.0 on smithi167 2024-08-31T08:25:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:25:53.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:25:50' 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:25:50' 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:25:50' 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:25:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:25:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142134294mirror.0smithi16719.3.0-4620-gdc9f1ccbtrueOK42254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":16741580.8,"bytes_per_snapshot":519456256.0,"last_snapshot_bytes":502247424,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"}42134294mirror.0smithi1672024-08-31 08:25:50cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:25:51' 2024-08-31T08:25:53.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:25:53.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:25:53.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:25:50' 2024-08-31T08:25:53.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:25:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:25:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:25:53.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:25:53.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap6 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap6 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:25:53.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:25:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:25:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6 2024-08-31T08:25:53.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:26:20.224 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7480796056118637) > weight (0.25) 2024-08-31T08:26:20.224 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.35245528075148536) > weight (0.25) 2024-08-31T08:26:20.224 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7173548165018054) > weight (0.25) 2024-08-31T08:26:20.224 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-08-31T08:26:20.225 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-08-31T08:26:20.225 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:26:20.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi167.stderr:2024-08-31T08:26:20.225+0000 7fcbad252640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 27072) UID: 1000 2024-08-31T08:26:20.322 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:26:20.323 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:26:20.323 INFO:tasks.rbd_mirror_thrash:waiting for 9 secs before reviving daemons 2024-08-31T08:26:20.324 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:26:20.323+0000 7fd8fcae7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 26933) UID: 1000 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:26:22.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:26:22.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:26:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:26:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:26:23.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:26:23.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:26:23.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:26:23.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:23.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:26:23.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:26:23.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:26:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:26:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:26:23.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:26:23.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:23.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:23.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:23.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:26:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-08-31T08:26:23.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:23.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:26:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:23.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:23.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:26:23.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:26:23.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:26:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:23.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:26:23.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:26:23.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:26:23.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:26:23.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:26:23.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:26:23.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:26:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:26:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:26:23.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:26:23.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:26:23.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:26:23.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2024-08-31T08:26:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:26:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:26:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:26:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:26:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:26:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:26:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:26:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:26:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:26:23.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2024-08-31T08:26:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:26:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:26:23.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:26:23.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:26:23.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 1 2024-08-31T08:26:24.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:26:24.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:24.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:26:24.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:26:24.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:26:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:26:24.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:26:24.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:26:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:26:24.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:26:24.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:26:24.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:26:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:26:24.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:26:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:26:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:26:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:26:24.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:26:24.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:26:26.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 20 2024-08-31T08:26:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:26:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:26.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:26:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:26:26.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:26:26.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:26:26.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=20 2024-08-31T08:26:26.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:26:26.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:26:26.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:26.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:26:26.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:26:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:26:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:26:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:26:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:26:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:26:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:26:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 20 = '' 2024-08-31T08:26:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:26:26.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:26:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:27.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:26:27.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:26:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:26:27.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:26:27.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 20 = '' 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:26:27.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:26:27.226 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-08-31T08:26:27.226 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:26:27.226 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-08-31T08:26:27.226 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-08-31T08:26:27.227 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-08-31T08:26:27.230 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-08-31T08:26:27.230 INFO:tasks.rbd_mirror_thrash:waiting for 101 secs before thrashing 2024-08-31T08:26:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:27.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:26:27.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:26:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:26:27.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:26:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:26:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:26:27.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:26:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:26:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 20 = '' 2024-08-31T08:26:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:26:27.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:26:29.324 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:26:29.324 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:26:29.324 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-08-31T08:26:29.324 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:26:29.325 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:26:29.327 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:26:29.327 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-08-31T08:26:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:26:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:26:29.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:26:29.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:26:29.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=20 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 20 = 20 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:26:29.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:26:30.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 12 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:26:30.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:24:49 2024 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:26:30.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap5-clean ']' 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap5-clean 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5-clean 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:30.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:26:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:26:31.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:31.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:26:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:32.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap6 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:26:32.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:26:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:33.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:26:33.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:33.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:26:33.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap6-clean 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap6-clean 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6-clean 2024-08-31T08:26:33.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:26:34.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:26:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:26:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:34.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:26:34.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:26:34.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:26:34.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:26:34.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 13529088 bytes 2024-08-31T08:26:35.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 92480512 bytes 2024-08-31T08:26:35.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 172014592 bytes 2024-08-31T08:26:35.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 209690624 bytes 2024-08-31T08:26:35.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 263128064 bytes 2024-08-31T08:26:35.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 287222784 bytes 2024-08-31T08:26:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 331328512 bytes 2024-08-31T08:26:36.296 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 387265536 bytes 2024-08-31T08:26:36.380 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 400057344 bytes 2024-08-31T08:26:36.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 436342784 bytes 2024-08-31T08:26:36.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 508394496 bytes 2024-08-31T08:26:36.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 523055104 bytes 2024-08-31T08:26:37.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 610056192 bytes 2024-08-31T08:26:37.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 645274624 bytes 2024-08-31T08:26:37.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 683367424 bytes 2024-08-31T08:26:37.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 705570816 bytes 2024-08-31T08:26:37.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 730971136 bytes 2024-08-31T08:26:37.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 783120384 bytes 2024-08-31T08:26:37.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 798269440 bytes 2024-08-31T08:26:38.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 871993344 bytes 2024-08-31T08:26:38.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 935506944 bytes 2024-08-31T08:26:38.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 960132096 bytes 2024-08-31T08:26:38.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1002902528 bytes 2024-08-31T08:26:38.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1045978112 bytes 2024-08-31T08:26:38.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1101223936 bytes 2024-08-31T08:26:38.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1126759424 bytes 2024-08-31T08:26:39.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1179057152 bytes 2024-08-31T08:26:39.279 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1225686016 bytes 2024-08-31T08:26:39.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1263200256 bytes 2024-08-31T08:26:39.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1305126912 bytes 2024-08-31T08:26:39.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1326580736 bytes 2024-08-31T08:26:39.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1358512128 bytes 2024-08-31T08:26:40.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1413452800 bytes 2024-08-31T08:26:40.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1463962624 bytes 2024-08-31T08:26:40.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1570476032 bytes 2024-08-31T08:26:40.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1597085696 bytes 2024-08-31T08:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1679785984 bytes 2024-08-31T08:26:40.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1704053760 bytes 2024-08-31T08:26:41.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1784679424 bytes 2024-08-31T08:26:41.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1848506368 bytes 2024-08-31T08:26:41.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1906033664 bytes 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:41.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:26:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:26:42.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:26:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:26:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:26:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:26:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"} 2024-08-31T08:26:42.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.1 on smithi167 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:26:24 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:26:21' 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:26:42.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:26:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:26:42.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"} 2024-08-31T08:26:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:26:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:26:42.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:26:42.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:26:42.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:26:42.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.1 on smithi167 2024-08-31T08:26:42.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:26:42.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:26:42.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:26:24' 2024-08-31T08:26:42.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:26:24' 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:26:24' 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:26:42.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:26:42.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbtrueOK45314534mirror.0smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725092731,"remote_snapshot_timestamp":1725092731,"replay_state":"idle"}44484451mirror.1smithi1672024-08-31 08:26:24cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:26:21' 2024-08-31T08:26:42.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:26:42.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:26:42.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:26:24' 2024-08-31T08:26:42.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:26:42.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:26:42.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:26:42.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:26:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:26:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:26:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap7 2024-08-31T08:26:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap7 2024-08-31T08:26:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:26:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:26:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:26:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7 2024-08-31T08:26:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:27:11.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:27:11.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:27:12.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:27:12.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:12.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:12.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:12.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:27:12.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:27:12.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:27:12.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:27:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:27:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:27:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:27:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:27:12.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:27:12.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:27:12.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:27:12.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:27:12.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:27:12.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:27:12.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:27:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:27:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:27:12.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:27:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:27:12.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:27:12.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:27:12.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:27:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:27:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:27:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:27:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:27:12.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:27:12.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:27:12.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:27:12.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:27:12.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:27:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:27:12.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:27:12.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:27:12.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:27:12.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:27:12.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:27:12.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:27:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:27:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:27:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:27:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:27:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:27:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:12.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:27:14.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 23 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:27:14.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:27:14.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:27:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=23 2024-08-31T08:27:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:27:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:27:14.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:27:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:14.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:27:14.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:27:14.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:27:14.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:27:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:27:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:27:14.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:27:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:27:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 23 = '' 2024-08-31T08:27:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:27:14.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:27:15.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:27:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:27:15.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:27:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:27:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:27:15.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:27:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:27:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 23 = '' 2024-08-31T08:27:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:27:15.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:15.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:27:15.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:27:16.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:27:16.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:27:16.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:27:16.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:27:16.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:27:16.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:27:16.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 23 = '' 2024-08-31T08:27:16.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:27:16.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:27:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:17.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:27:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:27:17.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:27:17.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:27:17.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=23 2024-08-31T08:27:17.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 23 = 23 2024-08-31T08:27:17.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:27:17.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:27:18.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2024-08-31T08:27:18.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:27:18.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:27:18.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:27:18.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 14 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:26:31 2024 2024-08-31T08:27:18.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:27:18.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap6-clean ']' 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap6-clean 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6-clean 2024-08-31T08:27:18.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:27:18.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:27:18.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:18.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:18.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:27:19.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:27:19.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:20.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:27:20.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:20.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap7 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:20.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:27:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:27:21.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:21.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:27:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:22.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:27:22.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap7-clean 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap7-clean 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7-clean 2024-08-31T08:27:22.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:27:22.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:27:22.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:27:22.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:27:22.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 48565248 bytes 2024-08-31T08:27:22.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 131976192 bytes 2024-08-31T08:27:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 209361920 bytes 2024-08-31T08:27:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 260500480 bytes 2024-08-31T08:27:23.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 300528640 bytes 2024-08-31T08:27:23.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 388284416 bytes 2024-08-31T08:27:23.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 413848576 bytes 2024-08-31T08:27:24.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 472155136 bytes 2024-08-31T08:27:24.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 530278400 bytes 2024-08-31T08:27:24.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 589993984 bytes 2024-08-31T08:27:24.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 640870400 bytes 2024-08-31T08:27:24.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 698831872 bytes 2024-08-31T08:27:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 733235200 bytes 2024-08-31T08:27:25.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 784097280 bytes 2024-08-31T08:27:25.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 808671232 bytes 2024-08-31T08:27:25.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 860192768 bytes 2024-08-31T08:27:25.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 867502080 bytes 2024-08-31T08:27:25.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 894344192 bytes 2024-08-31T08:27:25.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 962584576 bytes 2024-08-31T08:27:25.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 981681152 bytes 2024-08-31T08:27:25.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1010791424 bytes 2024-08-31T08:27:26.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1033366528 bytes 2024-08-31T08:27:26.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1054797824 bytes 2024-08-31T08:27:26.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1137024000 bytes 2024-08-31T08:27:26.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1196165120 bytes 2024-08-31T08:27:26.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1224900608 bytes 2024-08-31T08:27:26.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1270532096 bytes 2024-08-31T08:27:27.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1316731904 bytes 2024-08-31T08:27:27.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1337833472 bytes 2024-08-31T08:27:27.279 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1338452992 bytes 2024-08-31T08:27:27.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1437009920 bytes 2024-08-31T08:27:27.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1457556480 bytes 2024-08-31T08:27:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1463168000 bytes 2024-08-31T08:27:28.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1536570368 bytes 2024-08-31T08:27:28.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1625260032 bytes 2024-08-31T08:27:28.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1655708672 bytes 2024-08-31T08:27:28.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1691195392 bytes 2024-08-31T08:27:28.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1742339072 bytes 2024-08-31T08:27:28.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1780519936 bytes 2024-08-31T08:27:28.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1784967168 bytes 2024-08-31T08:27:29.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1815923712 bytes 2024-08-31T08:27:29.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1860379648 bytes 2024-08-31T08:27:29.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1879485440 bytes 2024-08-31T08:27:29.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1888945152 bytes 2024-08-31T08:27:29.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1936831488 bytes 2024-08-31T08:27:29.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1959912448 bytes 2024-08-31T08:27:29.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2010144768 bytes 2024-08-31T08:27:30.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2056007680 bytes 2024-08-31T08:27:30.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2090814464 bytes 2024-08-31T08:27:30.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2096319488 bytes 2024-08-31T08:27:30.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2110164992 bytes 2024-08-31T08:27:30.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2121183232 bytes 2024-08-31T08:27:30.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2148887552 bytes 2024-08-31T08:27:30.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2152868864 bytes 2024-08-31T08:27:30.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2203837440 bytes 2024-08-31T08:27:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2239353856 bytes 2024-08-31T08:27:31.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2277592064 bytes 2024-08-31T08:27:31.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2343822336 bytes 2024-08-31T08:27:31.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2409364480 bytes 2024-08-31T08:27:31.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2486460416 bytes 2024-08-31T08:27:31.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2520749056 bytes 2024-08-31T08:27:32.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2652408832 bytes 2024-08-31T08:27:32.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2701464576 bytes 2024-08-31T08:27:32.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2746556416 bytes 2024-08-31T08:27:32.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2800088064 bytes 2024-08-31T08:27:32.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2851437568 bytes 2024-08-31T08:27:33.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2929044480 bytes 2024-08-31T08:27:33.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2950700032 bytes 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:27:33.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:27:33.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:27:33.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:27:34.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:27:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":17514564.27,"bytes_per_snapshot":531059712.0,"last_snapshot_bytes":525436928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092833,"remote_snapshot_timestamp":1725092833,"replay_state":"idle"} 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.1 on smithi167 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:27:24 2024-08-31T08:27:34.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:27:21' 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:27:34.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:27:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:27:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":17514564.27,"bytes_per_snapshot":531059712.0,"last_snapshot_bytes":525436928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092833,"remote_snapshot_timestamp":1725092833,"replay_state":"idle"} 2024-08-31T08:27:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:27:34.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:27:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:27:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:27:34.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:27:34.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.1 on smithi167 2024-08-31T08:27:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:27:34.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:27:24' 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:27:24' 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:27:24' 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:27:34.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:27:34.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbtrueOK45314534mirror.0smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":17514564.27,"bytes_per_snapshot":531059712.0,"last_snapshot_bytes":525436928,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092833,"remote_snapshot_timestamp":1725092833,"replay_state":"idle"}44484451mirror.1smithi1672024-08-31 08:27:24cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:27:21' 2024-08-31T08:27:34.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:27:34.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:27:34.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:27:24' 2024-08-31T08:27:34.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:27:34.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:27:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:27:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:27:34.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:27:34.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:34.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap8 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap8 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8 2024-08-31T08:27:34.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:27:44.328 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:27:44.329 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:27:44.329 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-08-31T08:27:44.330 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:27:44.329+0000 7f6c4d1ec640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 31741) UID: 1000 2024-08-31T08:27:50.329 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:27:50.330 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:27:50.330 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-08-31T08:27:50.330 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:27:50.330 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:27:50.333 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:27:50.333 INFO:tasks.rbd_mirror_thrash:waiting for 114 secs before thrashing 2024-08-31T08:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:28:05.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:28:05.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:28:05.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:28:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:28:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:28:05.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:28:05.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:28:05.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:28:05.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:28:05.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:28:05.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:28:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:28:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:28:05.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:28:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:28:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:28:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:28:05.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:28:05.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:28:05.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:28:05.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:28:05.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:28:05.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:28:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:28:05.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:28:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:28:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:28:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:28:05.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:28:05.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:28:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:28:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:28:05.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:28:05.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:28:05.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:28:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:28:05.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:05.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:28:07.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 26 2024-08-31T08:28:07.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:28:07.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:07.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:07.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:07.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:28:07.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:28:07.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:28:07.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:28:07.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=26 2024-08-31T08:28:07.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:28:07.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:28:07.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:07.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:28:07.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:28:07.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:28:07.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:28:07.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:28:07.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:28:07.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:28:07.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:28:07.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 26 = '' 2024-08-31T08:28:07.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:28:07.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:28:08.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:28:08.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:28:08.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:28:08.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 26 = '' 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:28:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:28:08.231 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8969359389540663) > weight (0.25) 2024-08-31T08:28:08.231 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6909137514617263) > weight (0.25) 2024-08-31T08:28:08.231 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.9649133032802655) > weight (0.25) 2024-08-31T08:28:08.231 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7506139845844675) > weight (0.25) 2024-08-31T08:28:08.231 INFO:tasks.rbd_mirror_thrash:waiting for 114 secs before thrashing 2024-08-31T08:28:08.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:08.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:28:08.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:28:09.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:28:09.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:28:09.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:28:09.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:28:09.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:28:09.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:28:09.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 26 = '' 2024-08-31T08:28:09.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:28:09.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:28:10.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:28:10.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:28:10.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=26 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 26 = 26 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:28:10.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:28:11.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2024-08-31T08:28:11.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:28:11.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 16 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:26:31 2024 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:28:11.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:28:11.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:28:11.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:28:11.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:11.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap7-clean ']' 2024-08-31T08:28:11.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap7-clean 2024-08-31T08:28:11.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7-clean 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:11.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:28:12.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:28:12.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:13.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:28:13.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:13.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:13.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:13.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap8 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:13.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:28:14.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:28:14.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:14.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:28:14.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:15.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:28:15.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap8-clean 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap8-clean 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8-clean 2024-08-31T08:28:15.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:28:15.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:28:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:28:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:15.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:28:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=14 2024-08-31T08:28:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:28:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:28:15.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 12135424 bytes 2024-08-31T08:28:16.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 83917824 bytes 2024-08-31T08:28:16.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 135851008 bytes 2024-08-31T08:28:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 136454144 bytes 2024-08-31T08:28:16.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 202030080 bytes 2024-08-31T08:28:16.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 279784448 bytes 2024-08-31T08:28:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 311209984 bytes 2024-08-31T08:28:17.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 336595968 bytes 2024-08-31T08:28:17.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 375963648 bytes 2024-08-31T08:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 438307840 bytes 2024-08-31T08:28:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 438936576 bytes 2024-08-31T08:28:17.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 439553024 bytes 2024-08-31T08:28:17.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 546774016 bytes 2024-08-31T08:28:17.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 547385344 bytes 2024-08-31T08:28:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 548003840 bytes 2024-08-31T08:28:18.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 560275456 bytes 2024-08-31T08:28:18.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 560903168 bytes 2024-08-31T08:28:18.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 602612736 bytes 2024-08-31T08:28:18.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 678955008 bytes 2024-08-31T08:28:18.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 723478528 bytes 2024-08-31T08:28:18.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 797163520 bytes 2024-08-31T08:28:19.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 836211712 bytes 2024-08-31T08:28:19.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 868616192 bytes 2024-08-31T08:28:19.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 965891072 bytes 2024-08-31T08:28:19.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1012377600 bytes 2024-08-31T08:28:19.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1038839808 bytes 2024-08-31T08:28:19.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1103168512 bytes 2024-08-31T08:28:20.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1124852736 bytes 2024-08-31T08:28:20.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1138129920 bytes 2024-08-31T08:28:20.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1184186368 bytes 2024-08-31T08:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1192294400 bytes 2024-08-31T08:28:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1270284288 bytes 2024-08-31T08:28:20.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1285767168 bytes 2024-08-31T08:28:20.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1320553472 bytes 2024-08-31T08:28:21.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1386122240 bytes 2024-08-31T08:28:21.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1414287360 bytes 2024-08-31T08:28:21.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1503634432 bytes 2024-08-31T08:28:21.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1504253952 bytes 2024-08-31T08:28:21.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1565836288 bytes 2024-08-31T08:28:21.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1603343360 bytes 2024-08-31T08:28:22.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1673720832 bytes 2024-08-31T08:28:22.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1747280896 bytes 2024-08-31T08:28:22.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1808778240 bytes 2024-08-31T08:28:22.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1854822400 bytes 2024-08-31T08:28:22.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1939142656 bytes 2024-08-31T08:28:23.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1984908288 bytes 2024-08-31T08:28:23.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2016613376 bytes 2024-08-31T08:28:23.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2121750528 bytes 2024-08-31T08:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2141757440 bytes 2024-08-31T08:28:23.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2159609856 bytes 2024-08-31T08:28:23.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2235301888 bytes 2024-08-31T08:28:24.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2269421568 bytes 2024-08-31T08:28:24.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2353943552 bytes 2024-08-31T08:28:24.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2359485440 bytes 2024-08-31T08:28:24.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2404404224 bytes 2024-08-31T08:28:24.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2424978432 bytes 2024-08-31T08:28:24.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2465835008 bytes 2024-08-31T08:28:25.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2509922304 bytes 2024-08-31T08:28:25.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2563509248 bytes 2024-08-31T08:28:25.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2615460864 bytes 2024-08-31T08:28:25.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2630210560 bytes 2024-08-31T08:28:25.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2642341888 bytes 2024-08-31T08:28:25.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2665710592 bytes 2024-08-31T08:28:25.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2666304512 bytes 2024-08-31T08:28:25.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2684664832 bytes 2024-08-31T08:28:26.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2741366784 bytes 2024-08-31T08:28:26.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2791463936 bytes 2024-08-31T08:28:26.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2819354624 bytes 2024-08-31T08:28:26.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2834567168 bytes 2024-08-31T08:28:26.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2844536832 bytes 2024-08-31T08:28:26.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2892274688 bytes 2024-08-31T08:28:26.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2964739072 bytes 2024-08-31T08:28:27.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3024084992 bytes 2024-08-31T08:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3061701632 bytes 2024-08-31T08:28:27.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3100769280 bytes 2024-08-31T08:28:27.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3105937408 bytes 2024-08-31T08:28:27.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3118538752 bytes 2024-08-31T08:28:27.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3170507776 bytes 2024-08-31T08:28:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3208625152 bytes 2024-08-31T08:28:28.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3229648896 bytes 2024-08-31T08:28:28.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3279332352 bytes 2024-08-31T08:28:28.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8200 IOs, 3319727104 bytes 2024-08-31T08:28:28.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8300 IOs, 3353592832 bytes 2024-08-31T08:28:28.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8400 IOs, 3368016896 bytes 2024-08-31T08:28:28.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8500 IOs, 3433102336 bytes 2024-08-31T08:28:29.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8600 IOs, 3521056768 bytes 2024-08-31T08:28:29.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8700 IOs, 3557054464 bytes 2024-08-31T08:28:29.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8800 IOs, 3584057344 bytes 2024-08-31T08:28:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8900 IOs, 3635162112 bytes 2024-08-31T08:28:29.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:28:29.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:28:30.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:28:30.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:28:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:28:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:28:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:28:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":17765034.67,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"} 2024-08-31T08:28:30.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.1 on smithi167 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:28:24 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:28:21' 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:28:30.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:28:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:28:30.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":17765034.67,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"} 2024-08-31T08:28:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:28:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:28:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:28:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:28:30.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:28:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.1 on smithi167 2024-08-31T08:28:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:28:30.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:28:24' 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:28:24' 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:28:24' 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:28:30.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:28:30.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbtrueOK45864589mirror.0smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":17765034.67,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"}44484451mirror.1smithi1672024-08-31 08:28:24cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:28:21' 2024-08-31T08:28:30.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:28:30.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:28:30.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:28:24' 2024-08-31T08:28:30.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:28:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:28:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:28:30.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:28:30.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:28:30.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:30.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:28:30.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap9 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap9 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9 2024-08-31T08:28:30.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:28:59.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:28:59.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:28:59.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:29:00.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:29:00.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:29:00.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:29:00.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:29:00.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:00.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:29:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:29:00.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:29:00.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:29:00.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:29:00.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:29:00.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:29:00.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:29:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:29:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:29:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:29:00.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:29:00.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:00.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:00.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:00.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:29:00.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:29:00.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:29:00.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:29:00.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:29:00.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:29:00.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:29:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:00.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:00.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:29:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:29:00.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:29:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:29:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:29:00.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:00.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:29:00.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:29:00.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:00.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:29:02.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 29 2024-08-31T08:29:02.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:02.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:02.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:02.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:02.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:02.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:29:02.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:02.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:29:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=29 2024-08-31T08:29:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:02.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:29:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:02.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:02.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:02.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:02.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 29 = '' 2024-08-31T08:29:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:03.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 29 = '' 2024-08-31T08:29:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:03.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:29:03.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:03.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:04.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:04.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:04.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:04.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:04.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:04.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:04.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 29 = '' 2024-08-31T08:29:04.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:04.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:05.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:05.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:05.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=29 2024-08-31T08:29:05.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 29 = 29 2024-08-31T08:29:05.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:05.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2024-08-31T08:29:05.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:29:05.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:29:06.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 18 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:29:06.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:28:12 2024 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap8-clean ']' 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap8-clean 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8-clean 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:06.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:06.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:07.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:07.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:08.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:08.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap9 2024-08-31T08:29:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:08.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:09.142 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:09.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:09.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:09.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:10.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:10.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap9-clean 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap9-clean 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9-clean 2024-08-31T08:29:10.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror test 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:10.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:29:10.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:29:10.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:29:10.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2024-08-31T08:29:11.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 47337472 bytes 2024-08-31T08:29:11.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 47956992 bytes 2024-08-31T08:29:11.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 89029632 bytes 2024-08-31T08:29:11.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 156397568 bytes 2024-08-31T08:29:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 241647616 bytes 2024-08-31T08:29:11.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 282192896 bytes 2024-08-31T08:29:12.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 309944320 bytes 2024-08-31T08:29:12.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 381624320 bytes 2024-08-31T08:29:12.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 423957504 bytes 2024-08-31T08:29:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 465312768 bytes 2024-08-31T08:29:12.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 521989120 bytes 2024-08-31T08:29:12.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 562836480 bytes 2024-08-31T08:29:13.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 580871168 bytes 2024-08-31T08:29:13.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 655213568 bytes 2024-08-31T08:29:13.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 704847872 bytes 2024-08-31T08:29:13.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 723568640 bytes 2024-08-31T08:29:13.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 806144000 bytes 2024-08-31T08:29:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 835336192 bytes 2024-08-31T08:29:13.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 835927040 bytes 2024-08-31T08:29:14.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 896271360 bytes 2024-08-31T08:29:14.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 939203584 bytes 2024-08-31T08:29:14.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 951369728 bytes 2024-08-31T08:29:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 996992000 bytes 2024-08-31T08:29:14.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1081171968 bytes 2024-08-31T08:29:15.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1179766784 bytes 2024-08-31T08:29:15.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1226284032 bytes 2024-08-31T08:29:15.379 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1251450880 bytes 2024-08-31T08:29:15.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1288715264 bytes 2024-08-31T08:29:15.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1320583168 bytes 2024-08-31T08:29:15.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1361586176 bytes 2024-08-31T08:29:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1404807168 bytes 2024-08-31T08:29:16.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1420628992 bytes 2024-08-31T08:29:16.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1452872704 bytes 2024-08-31T08:29:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1503035392 bytes 2024-08-31T08:29:16.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1575428096 bytes 2024-08-31T08:29:16.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1637707776 bytes 2024-08-31T08:29:16.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1638312960 bytes 2024-08-31T08:29:17.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1694652416 bytes 2024-08-31T08:29:17.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1725763584 bytes 2024-08-31T08:29:17.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1755407360 bytes 2024-08-31T08:29:17.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1800679424 bytes 2024-08-31T08:29:17.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1848371200 bytes 2024-08-31T08:29:17.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1904388096 bytes 2024-08-31T08:29:18.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1932061696 bytes 2024-08-31T08:29:18.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1967102976 bytes 2024-08-31T08:29:18.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2021395456 bytes 2024-08-31T08:29:18.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2039011328 bytes 2024-08-31T08:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2123577344 bytes 2024-08-31T08:29:18.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2158625792 bytes 2024-08-31T08:29:19.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2191571968 bytes 2024-08-31T08:29:19.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2269891584 bytes 2024-08-31T08:29:19.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2270493696 bytes 2024-08-31T08:29:19.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2364824576 bytes 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:19.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:29:19.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local description_pattern= 2024-08-31T08:29:20.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local service_pattern= 2024-08-31T08:29:20.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:29:20.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:29:20.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='test: 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: global_id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+replaying 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"} 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: service: mirror.1 on smithi167 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:28:54 2024-08-31T08:29:20.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: peer_sites: 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: name: cluster2 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: state: up+stopped 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: description: local image is primary 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: last_update: 2024-08-31 08:28:51' 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ state: .*up+replaying' 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T08:29:20.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^ description: .*' 2024-08-31T08:29:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"} 2024-08-31T08:29:20.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:29:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:29:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep '^up+' 2024-08-31T08:29:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:29:20.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep 'service: *mirror..* on ' 2024-08-31T08:29:20.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.1 on smithi167 2024-08-31T08:29:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update 2024-08-31T08:29:20.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:28:54' 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2024-08-31 08:28:54' 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state_pattern=up+replaying 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local 'prev_last_update=2024-08-31 08:28:54' 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T08:29:20.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2024-08-31T08:29:20.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status='OKOKOK142254299mirror.3smithi16719.3.0-4620-gdc9f1ccbfalseOK43854388mirror.2smithi16719.3.0-4620-gdc9f1ccbfalseOK44484451mirror.1smithi16719.3.0-4620-gdc9f1ccbtrueOK45864589mirror.0smithi16719.3.0-4620-gdc9f1ccbfalseOKtest94efdf5f-e3d6-4d6c-9b64-cd319b6cf7aeup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":529193984.0,"last_snapshot_bytes":532951040,"last_snapshot_sync_seconds":1,"local_snapshot_timestamp":1725092886,"remote_snapshot_timestamp":1725092886,"replay_state":"idle"}44484451mirror.1smithi1672024-08-31 08:28:54cluster2b2ed8de6-469c-4293-9d02-39498e569bf2up+stoppedlocal image is primary2024-08-31 08:28:51' 2024-08-31T08:29:20.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_update state 2024-08-31T08:29:20.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2024-08-31T08:29:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ last_update='2024-08-31 08:28:54' 2024-08-31T08:29:20.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2024-08-31T08:29:20.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=up+replaying 2024-08-31T08:29:20.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo up+replaying 2024-08-31T08:29:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep up+replaying 2024-08-31T08:29:20.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:up+replaying 2024-08-31T08:29:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap10 2024-08-31T08:29:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap10 2024-08-31T08:29:20.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10 2024-08-31T08:29:20.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:29:44.334 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2889760073515515) > weight (0.25) 2024-08-31T08:29:44.334 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5988196458753591) > weight (0.25) 2024-08-31T08:29:44.334 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9960936130192516) > weight (0.25) 2024-08-31T08:29:44.334 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6858663708386162) > weight (0.25) 2024-08-31T08:29:44.335 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:50.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:50.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:29:50.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:29:50.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:29:50.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:29:51.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:29:51.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:29:51.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:29:51.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:29:51.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:29:51.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:51.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:29:51.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:29:51.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:29:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:29:51.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:29:51.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:29:51.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:29:51.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:29:51.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:29:51.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:29:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:29:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:29:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:29:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:29:51.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:29:51.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:51.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:51.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:29:51.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:29:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:29:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:29:51.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:29:51.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:29:51.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 32 2024-08-31T08:29:53.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:29:53.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:53.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:29:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=32 2024-08-31T08:29:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:53.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:29:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 32 = '' 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:53.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:29:54.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:54.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:54.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:54.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:54.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:54.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:54.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:54.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:54.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 32 = '' 2024-08-31T08:29:54.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:54.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:29:55.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:55.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:55.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:29:55.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:29:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:29:55.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:29:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:29:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 32 = '' 2024-08-31T08:29:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:29:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:29:56.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:29:56.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:29:56.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:29:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=32 2024-08-31T08:29:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 32 = 32 2024-08-31T08:29:56.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:29:56.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:29:57.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 20 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:29:57.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:28:12 2024 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:29:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snap9-clean ']' 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap9-clean 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9-clean 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:29:57.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:57.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:57.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:57.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:58.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:58.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:59.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:59.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap10 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:59.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:29:59.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:00.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:00.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:00.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:01.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ clean_snap_name=snap10-clean 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror test snap10-clean 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:01.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10-clean 2024-08-31T08:30:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2024-08-31T08:30:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror test 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:30:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:30:02.232 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5763273898924662) > weight (0.25) 2024-08-31T08:30:02.232 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:30:02.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:30:02.233 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-08-31T08:30:02.233 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:30:02.233+0000 7f3cf2d00640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 29455) UID: 1000 2024-08-31T08:30:02.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror test started 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:30:02.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:30:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:30:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:30:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:30:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:30:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:30:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:30:02.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:30:02.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:30:02.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:30:02.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:30:02.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:30:02.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:30:02.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:30:02.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:30:02.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:30:02.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:30:02.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2024-08-31T08:30:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/test'] 2024-08-31T08:30:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:30:02.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:30:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:30:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:30:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:30:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2024-08-31T08:30:02.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:30:02.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:30:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:30:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:30:02.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-test.status 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-test.status 2024-08-31T08:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-test.status 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-test.status 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror test 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2024-08-31T08:30:04.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 34 2024-08-31T08:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.status 2024-08-31T08:30:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:30:04.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2024-08-31T08:30:04.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:30:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=34 2024-08-31T08:30:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:30:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:30:04.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:30:05.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.status 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2024-08-31T08:30:05.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:30:05.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=34 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 34 = 34 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10-clean 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:30:05.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:30:06.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'test': 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 21 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 10c6565d3528 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.10c6565d3528 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:30:06.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:29:58 2024 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:21:22 2024 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae 2024-08-31T08:30:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:30:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 10 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap1 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap1 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:06.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:06.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:06.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:06.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:07.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:07.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:07.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap1-clean 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap1-clean 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap1-clean 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:07.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:08.233 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:30:08.234 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:30:08.234 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:30:08.234 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:30:08.234 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:30:08.237 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:30:08.237 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing 2024-08-31T08:30:08.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:08.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:09.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:09.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:09.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap2 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap2 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:09.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:10.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:10.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:11.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:11.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:11.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:11.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:11.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap2-clean 2024-08-31T08:30:11.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap2-clean 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap2-clean 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:11.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:12.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:12.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:12.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:12.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:12.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap3 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap3 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:13.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3 2024-08-31T08:30:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:13.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:13.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:13.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:14.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:14.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:14.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap3-clean 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap3-clean 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap3-clean 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:14.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:14.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:15.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:15.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:16.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:16.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap4 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap4 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:16.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:16.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:17.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:17.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:18.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:18.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:18.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap4-clean 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap4-clean 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap4-clean 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:18.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:18.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:18.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:19.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:19.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap5 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap5 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:20.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:20.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:20.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:20.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:21.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:21.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap5-clean 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap5-clean 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap5-clean 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:21.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:21.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:21.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:21.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:22.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:22.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:22.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:22.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap6 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap6 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:22.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:23.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:23.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:24.250 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:24.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:24.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:24.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap6-clean 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap6-clean 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap6-clean 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:25.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:25.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:25.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:25.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:26.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap7 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap7 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:26.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:26.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:26.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:26.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:27.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:27.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap7-clean 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap7-clean 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap7-clean 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:28.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:28.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:28.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:29.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:29.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:29.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap8 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap8 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8 2024-08-31T08:30:29.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:29.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:30.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:30.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:31.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:31.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap8-clean 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap8-clean 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:31.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap8-clean 2024-08-31T08:30:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:32.171 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:32.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:32.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:32.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:33.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:33.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:33.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:33.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap9 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap9 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:33.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:33.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:33.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:34.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap9-clean 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap9-clean 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap9-clean 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:34.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:34.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:35.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:35.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:36.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:36.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:36.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap10 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap10 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:36.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10 2024-08-31T08:30:36.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:36.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:36.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:36.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:37.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:37.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:37.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:37.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:38.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap10-clean 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror test snap10-clean 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap10-clean 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:30:38.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:38.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export 2024-08-31T08:30:38.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:39.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-08-31T08:30:39.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:39.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-test.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-test.export 2024-08-31T08:30:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:30:40.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 10 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap1 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap1 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap1 2024-08-31T08:30:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2024-08-31T08:30:40.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:40.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap1-clean 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap1-clean 2024-08-31T08:30:40.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2024-08-31T08:30:41.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap2 2024-08-31T08:30:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap2 2024-08-31T08:30:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap2 2024-08-31T08:30:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2024-08-31T08:30:42.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap2-clean 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:42.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap2-clean 2024-08-31T08:30:42.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2024-08-31T08:30:43.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap3 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap3 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap3 2024-08-31T08:30:43.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2024-08-31T08:30:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap3-clean 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap3-clean 2024-08-31T08:30:44.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2024-08-31T08:30:45.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:45.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap4 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap4 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap4 2024-08-31T08:30:45.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2024-08-31T08:30:46.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:46.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap4-clean 2024-08-31T08:30:46.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2024-08-31T08:30:46.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:46.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:46.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:46.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap4-clean 2024-08-31T08:30:46.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2024-08-31T08:30:47.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap5 2024-08-31T08:30:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap5 2024-08-31T08:30:47.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap5 2024-08-31T08:30:47.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2024-08-31T08:30:48.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap5-clean 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap5-clean 2024-08-31T08:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2024-08-31T08:30:49.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap6 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap6 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap6 2024-08-31T08:30:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2024-08-31T08:30:50.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap6-clean 2024-08-31T08:30:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2024-08-31T08:30:50.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap6-clean 2024-08-31T08:30:50.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2024-08-31T08:30:51.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap7 2024-08-31T08:30:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap7 2024-08-31T08:30:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:51.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:51.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap7 2024-08-31T08:30:51.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2024-08-31T08:30:52.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap7-clean 2024-08-31T08:30:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2024-08-31T08:30:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap7-clean 2024-08-31T08:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2024-08-31T08:30:53.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap8 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap8 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap8 2024-08-31T08:30:53.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2024-08-31T08:30:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap8-clean 2024-08-31T08:30:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2024-08-31T08:30:54.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:54.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:54.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:54.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap8-clean 2024-08-31T08:30:54.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2024-08-31T08:30:55.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap9 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap9 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap9 2024-08-31T08:30:55.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2024-08-31T08:30:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:56.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap9-clean 2024-08-31T08:30:56.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2024-08-31T08:30:56.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:56.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:56.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:56.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap9-clean 2024-08-31T08:30:56.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2024-08-31T08:30:57.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap10 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap10 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap10 2024-08-31T08:30:57.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2024-08-31T08:30:58.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap10-clean 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap10-clean 2024-08-31T08:30:58.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2024-08-31T08:30:59.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror test 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:30:59.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:30:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror test 2024-08-31T08:30:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:30:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:30:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:30:59.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2024-08-31T08:30:59.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/test 2024-08-31T08:30:59.899 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:30:59.898+0000 7f359c132640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55a1f775d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:30:59.899 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:30:59.898+0000 7f35a293f640 -1 rbd::mirror::ImageReplayer: 0x55a1f6c44000 [3/94efdf5f-e3d6-4d6c-9b64-cd319b6cf7ae] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:31:02.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:31:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:02.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror test deleted 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror test 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=test 2024-08-31T08:31:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-08-31T08:31:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:31:02.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image test: (2) No such file or directory 2024-08-31T08:31:02.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:31:02.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:31:02.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror test deleted '' 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=test 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:31:02.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror test 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=test 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/test 2024-08-31T08:31:02.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:31:02.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image test: (2) No such file or directory 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: create many images' 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:31:02.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:31:02.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:31:02 TEST: create many images 2024-08-31T08:31:02.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:31:02 TEST: create many images 2024-08-31T08:31:02.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ snap_name=snap 2024-08-31T08:31:02.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 50 2024-08-31T08:31:02.254 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_1 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_1 128M 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:02.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:02.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_1 2024-08-31T08:31:02.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:02.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:02.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-08-31T08:31:02.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:02.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-08-31T08:31:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_1': 2024-08-31T08:31:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:02.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1272ed1ca2b0 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1272ed1ca2b0 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:02 2024 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:02 2024 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:02 2024 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 7317f9a4-8aa2-4900-b19f-fff053c553f4 2024-08-31T08:31:02.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_1 2024-08-31T08:31:02.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:02.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:02.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:31:02.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:02.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:31:02.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:02.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:03.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 52004864 bytes 2024-08-31T08:31:03.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 113054720 bytes 2024-08-31T08:31:03.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 177488896 bytes 2024-08-31T08:31:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 280146944 bytes 2024-08-31T08:31:03.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 322861056 bytes 2024-08-31T08:31:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 374018048 bytes 2024-08-31T08:31:04.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 421140480 bytes 2024-08-31T08:31:04.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 440433664 bytes 2024-08-31T08:31:04.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 459927552 bytes 2024-08-31T08:31:04.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 497093632 bytes 2024-08-31T08:31:04.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 518580224 bytes 2024-08-31T08:31:04.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 548786176 bytes 2024-08-31T08:31:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 605068288 bytes 2024-08-31T08:31:05.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 638025728 bytes 2024-08-31T08:31:05.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 645497856 bytes 2024-08-31T08:31:05.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 711334912 bytes 2024-08-31T08:31:05.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 735550464 bytes 2024-08-31T08:31:05.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 737799168 bytes 2024-08-31T08:31:05.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 764647424 bytes 2024-08-31T08:31:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 784379904 bytes 2024-08-31T08:31:06.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 847702016 bytes 2024-08-31T08:31:06.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 888630272 bytes 2024-08-31T08:31:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 925745152 bytes 2024-08-31T08:31:06.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 945515520 bytes 2024-08-31T08:31:06.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 986305536 bytes 2024-08-31T08:31:06.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1022418944 bytes 2024-08-31T08:31:07.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1040868352 bytes 2024-08-31T08:31:07.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1048824832 bytes 2024-08-31T08:31:07.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1116971008 bytes 2024-08-31T08:31:07.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1193277440 bytes 2024-08-31T08:31:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1289996288 bytes 2024-08-31T08:31:08.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1370206208 bytes 2024-08-31T08:31:08.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1396415488 bytes 2024-08-31T08:31:08.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1402882048 bytes 2024-08-31T08:31:08.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1452441600 bytes 2024-08-31T08:31:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1527701504 bytes 2024-08-31T08:31:08.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1592346624 bytes 2024-08-31T08:31:09.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1659020288 bytes 2024-08-31T08:31:09.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8273976210576359) > weight (0.25) 2024-08-31T08:31:09.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5715760990796906) > weight (0.25) 2024-08-31T08:31:09.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5504096820462214) > weight (0.25) 2024-08-31T08:31:09.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.9622955227607481) > weight (0.25) 2024-08-31T08:31:09.239 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing 2024-08-31T08:31:09.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1703309312 bytes 2024-08-31T08:31:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1703915520 bytes 2024-08-31T08:31:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1721827328 bytes 2024-08-31T08:31:09.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1760718848 bytes 2024-08-31T08:31:09.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1778434048 bytes 2024-08-31T08:31:09.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1819357184 bytes 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:09.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_2 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_2 128M 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:09.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:09.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_2 2024-08-31T08:31:09.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:09.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:09.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-08-31T08:31:10.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:10.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_2': 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 127e38afa2e7 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.127e38afa2e7 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:09 2024 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:09 2024 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:09 2024 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b9749456-99a0-4f9d-86ad-1654ff6a6a36 2024-08-31T08:31:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_2 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:31:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:31:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:10.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:11.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 689152 bytes 2024-08-31T08:31:11.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 34439168 bytes 2024-08-31T08:31:11.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 55742464 bytes 2024-08-31T08:31:11.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 81665024 bytes 2024-08-31T08:31:11.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 111255552 bytes 2024-08-31T08:31:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 150025216 bytes 2024-08-31T08:31:11.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 163715072 bytes 2024-08-31T08:31:12.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 222360576 bytes 2024-08-31T08:31:12.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 235463680 bytes 2024-08-31T08:31:12.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 311329792 bytes 2024-08-31T08:31:12.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 432760832 bytes 2024-08-31T08:31:12.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 505042944 bytes 2024-08-31T08:31:13.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 553763840 bytes 2024-08-31T08:31:13.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 617008128 bytes 2024-08-31T08:31:13.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 641880064 bytes 2024-08-31T08:31:13.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 672035840 bytes 2024-08-31T08:31:13.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 694714368 bytes 2024-08-31T08:31:13.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 777849856 bytes 2024-08-31T08:31:14.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 864292864 bytes 2024-08-31T08:31:14.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 879952896 bytes 2024-08-31T08:31:14.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 940976128 bytes 2024-08-31T08:31:14.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 966430720 bytes 2024-08-31T08:31:14.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1038252032 bytes 2024-08-31T08:31:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1071755264 bytes 2024-08-31T08:31:15.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1097208832 bytes 2024-08-31T08:31:15.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1123746816 bytes 2024-08-31T08:31:15.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1147031552 bytes 2024-08-31T08:31:15.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1211573248 bytes 2024-08-31T08:31:15.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1247129600 bytes 2024-08-31T08:31:15.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1319424000 bytes 2024-08-31T08:31:16.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1394612224 bytes 2024-08-31T08:31:16.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1412469760 bytes 2024-08-31T08:31:16.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1413072896 bytes 2024-08-31T08:31:16.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1501880320 bytes 2024-08-31T08:31:16.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1568561152 bytes 2024-08-31T08:31:16.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1575001088 bytes 2024-08-31T08:31:17.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1655298048 bytes 2024-08-31T08:31:17.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1703667712 bytes 2024-08-31T08:31:17.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1724573696 bytes 2024-08-31T08:31:17.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1778271232 bytes 2024-08-31T08:31:17.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1821469696 bytes 2024-08-31T08:31:17.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1864957952 bytes 2024-08-31T08:31:18.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1965307904 bytes 2024-08-31T08:31:18.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2026821632 bytes 2024-08-31T08:31:18.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2108397568 bytes 2024-08-31T08:31:18.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2142903296 bytes 2024-08-31T08:31:18.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2180852736 bytes 2024-08-31T08:31:18.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2187251712 bytes 2024-08-31T08:31:19.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2221148160 bytes 2024-08-31T08:31:19.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2260892672 bytes 2024-08-31T08:31:19.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2297971712 bytes 2024-08-31T08:31:19.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2327931904 bytes 2024-08-31T08:31:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2395219968 bytes 2024-08-31T08:31:19.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2459902976 bytes 2024-08-31T08:31:20.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2502278144 bytes 2024-08-31T08:31:20.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2526578688 bytes 2024-08-31T08:31:20.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2574657536 bytes 2024-08-31T08:31:20.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2610416640 bytes 2024-08-31T08:31:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2630739968 bytes 2024-08-31T08:31:20.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2677936128 bytes 2024-08-31T08:31:21.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2714219520 bytes 2024-08-31T08:31:21.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2749775872 bytes 2024-08-31T08:31:21.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2782258176 bytes 2024-08-31T08:31:21.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2844168192 bytes 2024-08-31T08:31:21.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2869053440 bytes 2024-08-31T08:31:21.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2894476288 bytes 2024-08-31T08:31:21.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2929021952 bytes 2024-08-31T08:31:22.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2941917184 bytes 2024-08-31T08:31:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2977178624 bytes 2024-08-31T08:31:22.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3006332928 bytes 2024-08-31T08:31:22.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3034822656 bytes 2024-08-31T08:31:22.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3063020544 bytes 2024-08-31T08:31:22.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3079346176 bytes 2024-08-31T08:31:23.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_3 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_3 128M 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:23.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_3 2024-08-31T08:31:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:23.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:23.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-08-31T08:31:23.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:23.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-08-31T08:31:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_3': 2024-08-31T08:31:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:23.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 128a5ed20a68 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.128a5ed20a68 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:23 2024 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:23 2024 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:23 2024 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c6165d76-3f8e-4fae-95c8-1c9ff3d6281d 2024-08-31T08:31:23.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:23.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_3 2024-08-31T08:31:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:31:23.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:31:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:23.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 81579008 bytes 2024-08-31T08:31:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 144048128 bytes 2024-08-31T08:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 180740096 bytes 2024-08-31T08:31:24.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 216551424 bytes 2024-08-31T08:31:24.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 263592960 bytes 2024-08-31T08:31:24.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 288210944 bytes 2024-08-31T08:31:24.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 317597696 bytes 2024-08-31T08:31:24.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 350693376 bytes 2024-08-31T08:31:24.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 395189248 bytes 2024-08-31T08:31:25.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 443922432 bytes 2024-08-31T08:31:25.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 451362816 bytes 2024-08-31T08:31:25.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 473634816 bytes 2024-08-31T08:31:25.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 474265600 bytes 2024-08-31T08:31:25.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 542265344 bytes 2024-08-31T08:31:25.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 566222848 bytes 2024-08-31T08:31:25.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 636193792 bytes 2024-08-31T08:31:26.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 673690624 bytes 2024-08-31T08:31:26.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 696474624 bytes 2024-08-31T08:31:26.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9380755430888851) > weight (0.25) 2024-08-31T08:31:26.335 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8040537324999733) > weight (0.25) 2024-08-31T08:31:26.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.35496886976883446) > weight (0.25) 2024-08-31T08:31:26.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7934218359875076) > weight (0.25) 2024-08-31T08:31:26.336 INFO:tasks.rbd_mirror_thrash:waiting for 84 secs before thrashing 2024-08-31T08:31:26.380 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 719204352 bytes 2024-08-31T08:31:26.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 750036992 bytes 2024-08-31T08:31:26.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 791804928 bytes 2024-08-31T08:31:26.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 792410112 bytes 2024-08-31T08:31:26.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 833299456 bytes 2024-08-31T08:31:27.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 891737088 bytes 2024-08-31T08:31:27.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 927262720 bytes 2024-08-31T08:31:27.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 952359936 bytes 2024-08-31T08:31:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 983584768 bytes 2024-08-31T08:31:27.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 986252288 bytes 2024-08-31T08:31:27.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1061032960 bytes 2024-08-31T08:31:27.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1085294592 bytes 2024-08-31T08:31:28.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1096178688 bytes 2024-08-31T08:31:28.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1098240000 bytes 2024-08-31T08:31:28.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1141955584 bytes 2024-08-31T08:31:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1142588416 bytes 2024-08-31T08:31:28.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1214048256 bytes 2024-08-31T08:31:28.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1228761088 bytes 2024-08-31T08:31:28.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1297610752 bytes 2024-08-31T08:31:29.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1408924672 bytes 2024-08-31T08:31:29.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1472894976 bytes 2024-08-31T08:31:29.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1526864896 bytes 2024-08-31T08:31:29.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1527478272 bytes 2024-08-31T08:31:29.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1541652480 bytes 2024-08-31T08:31:30.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1590074368 bytes 2024-08-31T08:31:30.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1635658752 bytes 2024-08-31T08:31:30.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1694411776 bytes 2024-08-31T08:31:30.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1760530432 bytes 2024-08-31T08:31:30.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1793234944 bytes 2024-08-31T08:31:31.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1834449920 bytes 2024-08-31T08:31:31.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1855529984 bytes 2024-08-31T08:31:31.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1892843520 bytes 2024-08-31T08:31:31.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1970078720 bytes 2024-08-31T08:31:31.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2011351040 bytes 2024-08-31T08:31:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2085314560 bytes 2024-08-31T08:31:32.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2093284352 bytes 2024-08-31T08:31:32.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2125701120 bytes 2024-08-31T08:31:32.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2228525056 bytes 2024-08-31T08:31:32.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2277331968 bytes 2024-08-31T08:31:32.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2290365440 bytes 2024-08-31T08:31:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2356621312 bytes 2024-08-31T08:31:33.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2365037568 bytes 2024-08-31T08:31:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2396606464 bytes 2024-08-31T08:31:33.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2425279488 bytes 2024-08-31T08:31:33.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2428963840 bytes 2024-08-31T08:31:33.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2518127616 bytes 2024-08-31T08:31:33.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2530853888 bytes 2024-08-31T08:31:33.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2558639104 bytes 2024-08-31T08:31:34.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2632888320 bytes 2024-08-31T08:31:34.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2665829376 bytes 2024-08-31T08:31:34.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2719820800 bytes 2024-08-31T08:31:34.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2760600576 bytes 2024-08-31T08:31:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2808352768 bytes 2024-08-31T08:31:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2856207360 bytes 2024-08-31T08:31:35.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 2860683264 bytes 2024-08-31T08:31:35.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 2915929088 bytes 2024-08-31T08:31:35.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 2917983232 bytes 2024-08-31T08:31:35.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 2938938368 bytes 2024-08-31T08:31:35.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 2978968576 bytes 2024-08-31T08:31:35.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3055431680 bytes 2024-08-31T08:31:35.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3062067200 bytes 2024-08-31T08:31:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3118963712 bytes 2024-08-31T08:31:36.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3152372736 bytes 2024-08-31T08:31:36.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8200 IOs, 3154035712 bytes 2024-08-31T08:31:36.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8300 IOs, 3206808576 bytes 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_4 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2024-08-31T08:31:36.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_4 128M 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:36.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:36.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_4 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-08-31T08:31:37.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:37.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-08-31T08:31:37.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_4': 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1296cf0fe0e4 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1296cf0fe0e4 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:36 2024 2024-08-31T08:31:37.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:36 2024 2024-08-31T08:31:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:36 2024 2024-08-31T08:31:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6227f895-18e3-43af-ab10-bcfa84946b75 2024-08-31T08:31:37.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:37.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:37.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_4 2024-08-31T08:31:37.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:37.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:37.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:31:37.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:37.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:31:37.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:37.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:37.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 137678848 bytes 2024-08-31T08:31:38.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 218339328 bytes 2024-08-31T08:31:38.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 278668288 bytes 2024-08-31T08:31:38.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 351270912 bytes 2024-08-31T08:31:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 403491840 bytes 2024-08-31T08:31:39.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 422928384 bytes 2024-08-31T08:31:39.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 470970368 bytes 2024-08-31T08:31:39.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 506146816 bytes 2024-08-31T08:31:39.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 524833792 bytes 2024-08-31T08:31:39.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 577152000 bytes 2024-08-31T08:31:39.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 614141952 bytes 2024-08-31T08:31:39.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 628782080 bytes 2024-08-31T08:31:39.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 636141568 bytes 2024-08-31T08:31:40.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 689923072 bytes 2024-08-31T08:31:40.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 720094208 bytes 2024-08-31T08:31:40.434 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 769324032 bytes 2024-08-31T08:31:40.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 816447488 bytes 2024-08-31T08:31:40.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 906838016 bytes 2024-08-31T08:31:40.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 907449344 bytes 2024-08-31T08:31:41.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 971140096 bytes 2024-08-31T08:31:41.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1028420608 bytes 2024-08-31T08:31:41.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1069117440 bytes 2024-08-31T08:31:41.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1098238976 bytes 2024-08-31T08:31:41.584 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:31:41.583+0000 7f6cb0353640 -1 reset not still connected to 0x5599524a5ee0 2024-08-31T08:31:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1149385728 bytes 2024-08-31T08:31:41.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1221163008 bytes 2024-08-31T08:31:42.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1268631552 bytes 2024-08-31T08:31:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1293664256 bytes 2024-08-31T08:31:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1350241280 bytes 2024-08-31T08:31:42.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1391335424 bytes 2024-08-31T08:31:42.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1436707840 bytes 2024-08-31T08:31:43.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1488353280 bytes 2024-08-31T08:31:43.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1517887488 bytes 2024-08-31T08:31:43.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1578227712 bytes 2024-08-31T08:31:43.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1615898624 bytes 2024-08-31T08:31:43.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1696328704 bytes 2024-08-31T08:31:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1730583552 bytes 2024-08-31T08:31:43.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1731194880 bytes 2024-08-31T08:31:44.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1777069056 bytes 2024-08-31T08:31:44.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1812662272 bytes 2024-08-31T08:31:44.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1877028864 bytes 2024-08-31T08:31:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1898908672 bytes 2024-08-31T08:31:44.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1919572992 bytes 2024-08-31T08:31:44.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1938073600 bytes 2024-08-31T08:31:45.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2037314560 bytes 2024-08-31T08:31:45.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2106200064 bytes 2024-08-31T08:31:45.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2109716480 bytes 2024-08-31T08:31:45.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2118554624 bytes 2024-08-31T08:31:45.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2154343424 bytes 2024-08-31T08:31:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2244257792 bytes 2024-08-31T08:31:46.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2244876288 bytes 2024-08-31T08:31:46.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2300034048 bytes 2024-08-31T08:31:46.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2330556416 bytes 2024-08-31T08:31:46.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2367511552 bytes 2024-08-31T08:31:46.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2448955392 bytes 2024-08-31T08:31:47.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2532020224 bytes 2024-08-31T08:31:47.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2610607104 bytes 2024-08-31T08:31:47.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2701314048 bytes 2024-08-31T08:31:47.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2735264768 bytes 2024-08-31T08:31:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2754015232 bytes 2024-08-31T08:31:48.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2808400896 bytes 2024-08-31T08:31:48.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2852921344 bytes 2024-08-31T08:31:48.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2938649600 bytes 2024-08-31T08:31:48.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2939293696 bytes 2024-08-31T08:31:48.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2949043200 bytes 2024-08-31T08:31:48.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 3000295424 bytes 2024-08-31T08:31:49.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 3110317056 bytes 2024-08-31T08:31:49.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 3140251648 bytes 2024-08-31T08:31:49.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 3201926144 bytes 2024-08-31T08:31:49.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 3223040000 bytes 2024-08-31T08:31:49.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3285763072 bytes 2024-08-31T08:31:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3370481664 bytes 2024-08-31T08:31:50.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3416561664 bytes 2024-08-31T08:31:50.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3439043584 bytes 2024-08-31T08:31:50.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3489446912 bytes 2024-08-31T08:31:50.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_5 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_5 128M 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:50.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:50.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_5 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2024-08-31T08:31:50.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:50.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:50.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:31:50.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:50.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-08-31T08:31:51.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:51.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_5': 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12a2c6b932c3 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12a2c6b932c3 2024-08-31T08:31:51.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:50 2024 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:50 2024 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:50 2024 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 72388a70-106e-4eb6-a7f8-822bc6c40054 2024-08-31T08:31:51.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:51.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:51.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_5 2024-08-31T08:31:51.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:51.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:51.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:31:51.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:51.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:31:51.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:51.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:51.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 59987968 bytes 2024-08-31T08:31:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 85512192 bytes 2024-08-31T08:31:52.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 105091072 bytes 2024-08-31T08:31:52.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 114422784 bytes 2024-08-31T08:31:52.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 120284160 bytes 2024-08-31T08:31:52.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 120910848 bytes 2024-08-31T08:31:52.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 170480640 bytes 2024-08-31T08:31:52.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 244367360 bytes 2024-08-31T08:31:52.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 256765952 bytes 2024-08-31T08:31:53.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 336646144 bytes 2024-08-31T08:31:53.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 337254400 bytes 2024-08-31T08:31:53.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 401936384 bytes 2024-08-31T08:31:53.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 454052864 bytes 2024-08-31T08:31:53.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 503729152 bytes 2024-08-31T08:31:53.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 532307968 bytes 2024-08-31T08:31:54.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 571447296 bytes 2024-08-31T08:31:54.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 620424192 bytes 2024-08-31T08:31:54.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 701873152 bytes 2024-08-31T08:31:54.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 735478784 bytes 2024-08-31T08:31:54.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 791766016 bytes 2024-08-31T08:31:54.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 809693184 bytes 2024-08-31T08:31:55.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 840643584 bytes 2024-08-31T08:31:55.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 926929920 bytes 2024-08-31T08:31:55.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 995272704 bytes 2024-08-31T08:31:55.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 995881984 bytes 2024-08-31T08:31:55.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1044761600 bytes 2024-08-31T08:31:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1064583168 bytes 2024-08-31T08:31:56.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1110663168 bytes 2024-08-31T08:31:56.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1140284416 bytes 2024-08-31T08:31:56.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1168695296 bytes 2024-08-31T08:31:56.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1216324608 bytes 2024-08-31T08:31:56.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1271479296 bytes 2024-08-31T08:31:56.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1338043392 bytes 2024-08-31T08:31:57.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1389177856 bytes 2024-08-31T08:31:57.246 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1449954304 bytes 2024-08-31T08:31:57.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1450561536 bytes 2024-08-31T08:31:57.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1504518144 bytes 2024-08-31T08:31:57.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1558858752 bytes 2024-08-31T08:31:57.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1571910656 bytes 2024-08-31T08:31:58.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1629004800 bytes 2024-08-31T08:31:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1722030080 bytes 2024-08-31T08:31:58.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1756822528 bytes 2024-08-31T08:31:58.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1805974528 bytes 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_6 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2024-08-31T08:31:58.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_6 128M 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:58.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:31:58.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_6 2024-08-31T08:31:58.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:31:58.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-08-31T08:31:59.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:31:59.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_6': 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12aecb4dfdb4 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12aecb4dfdb4 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:31:59.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:58 2024 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:58 2024 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:58 2024 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 9f6764b3-49ac-434c-9bd1-5079abbddbf0 2024-08-31T08:31:59.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:31:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:31:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_6 2024-08-31T08:31:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:31:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:31:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:31:59.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:31:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:31:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:31:59.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:31:59.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 44813312 bytes 2024-08-31T08:32:00.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 86652928 bytes 2024-08-31T08:32:00.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 166460416 bytes 2024-08-31T08:32:00.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 258096128 bytes 2024-08-31T08:32:00.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 258702336 bytes 2024-08-31T08:32:00.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 283811840 bytes 2024-08-31T08:32:00.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 299016192 bytes 2024-08-31T08:32:01.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 353714176 bytes 2024-08-31T08:32:01.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 365855744 bytes 2024-08-31T08:32:01.254 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 402301952 bytes 2024-08-31T08:32:01.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 458412032 bytes 2024-08-31T08:32:01.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 507516928 bytes 2024-08-31T08:32:01.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 593153024 bytes 2024-08-31T08:32:01.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 608591872 bytes 2024-08-31T08:32:02.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 676315136 bytes 2024-08-31T08:32:02.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 703064064 bytes 2024-08-31T08:32:02.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 754007040 bytes 2024-08-31T08:32:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 788337664 bytes 2024-08-31T08:32:02.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 854583296 bytes 2024-08-31T08:32:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 872328192 bytes 2024-08-31T08:32:03.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 967416832 bytes 2024-08-31T08:32:03.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1001019392 bytes 2024-08-31T08:32:03.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1019116544 bytes 2024-08-31T08:32:03.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1091278848 bytes 2024-08-31T08:32:04.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1191240704 bytes 2024-08-31T08:32:04.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1237178368 bytes 2024-08-31T08:32:04.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1271718912 bytes 2024-08-31T08:32:04.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1330951168 bytes 2024-08-31T08:32:04.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1418417152 bytes 2024-08-31T08:32:05.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1458716672 bytes 2024-08-31T08:32:05.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1548408832 bytes 2024-08-31T08:32:05.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1601483776 bytes 2024-08-31T08:32:05.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1653893120 bytes 2024-08-31T08:32:05.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1695021056 bytes 2024-08-31T08:32:06.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1764523008 bytes 2024-08-31T08:32:06.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1780464640 bytes 2024-08-31T08:32:06.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1821017088 bytes 2024-08-31T08:32:06.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1859835904 bytes 2024-08-31T08:32:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1922278400 bytes 2024-08-31T08:32:06.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1944060928 bytes 2024-08-31T08:32:07.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1979428864 bytes 2024-08-31T08:32:07.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 2052625408 bytes 2024-08-31T08:32:07.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2105917440 bytes 2024-08-31T08:32:07.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2106521600 bytes 2024-08-31T08:32:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2146032640 bytes 2024-08-31T08:32:07.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2182461440 bytes 2024-08-31T08:32:08.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2231127040 bytes 2024-08-31T08:32:08.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2308264960 bytes 2024-08-31T08:32:08.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2407553024 bytes 2024-08-31T08:32:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:32:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:32:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:32:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_7 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_7 128M 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:08.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_7 2024-08-31T08:32:08.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:32:08.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:08.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-08-31T08:32:09.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:32:09.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-08-31T08:32:09.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_7': 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12ba7e83a33b 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12ba7e83a33b 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:08 2024 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:08 2024 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:08 2024 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:32:09.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:32:09.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: be6ad6c7-3828-4fb6-ad59-75775c1277eb 2024-08-31T08:32:09.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_7 2024-08-31T08:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:32:09.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:32:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:32:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:32:09.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:32:09.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 31421440 bytes 2024-08-31T08:32:09.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 83917824 bytes 2024-08-31T08:32:09.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 105651200 bytes 2024-08-31T08:32:10.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 222680064 bytes 2024-08-31T08:32:10.240 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5006154218061182) > weight (0.25) 2024-08-31T08:32:10.240 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.26488505777180904) > weight (0.25) 2024-08-31T08:32:10.240 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5251948661281995) > weight (0.25) 2024-08-31T08:32:10.240 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8673931925589581) > weight (0.25) 2024-08-31T08:32:10.240 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-08-31T08:32:10.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 284010496 bytes 2024-08-31T08:32:10.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 322604032 bytes 2024-08-31T08:32:10.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 367514624 bytes 2024-08-31T08:32:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 420781056 bytes 2024-08-31T08:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 471265280 bytes 2024-08-31T08:32:11.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 493265920 bytes 2024-08-31T08:32:11.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 538043392 bytes 2024-08-31T08:32:11.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 556866560 bytes 2024-08-31T08:32:11.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 621022208 bytes 2024-08-31T08:32:11.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 636868608 bytes 2024-08-31T08:32:11.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 655894528 bytes 2024-08-31T08:32:12.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 720708608 bytes 2024-08-31T08:32:12.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 721337344 bytes 2024-08-31T08:32:12.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 721954816 bytes 2024-08-31T08:32:12.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 727835648 bytes 2024-08-31T08:32:12.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 749539328 bytes 2024-08-31T08:32:12.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 796588032 bytes 2024-08-31T08:32:12.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 830769152 bytes 2024-08-31T08:32:12.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 891439104 bytes 2024-08-31T08:32:13.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 942929920 bytes 2024-08-31T08:32:13.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 966092800 bytes 2024-08-31T08:32:13.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 993855488 bytes 2024-08-31T08:32:13.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1031750656 bytes 2024-08-31T08:32:13.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1045602304 bytes 2024-08-31T08:32:13.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1168693248 bytes 2024-08-31T08:32:14.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1252635648 bytes 2024-08-31T08:32:14.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1306165248 bytes 2024-08-31T08:32:14.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1352525824 bytes 2024-08-31T08:32:14.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1369671680 bytes 2024-08-31T08:32:14.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1396381696 bytes 2024-08-31T08:32:14.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1442441216 bytes 2024-08-31T08:32:15.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1465698304 bytes 2024-08-31T08:32:15.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1555135488 bytes 2024-08-31T08:32:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1605742592 bytes 2024-08-31T08:32:15.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1661996032 bytes 2024-08-31T08:32:15.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1686360064 bytes 2024-08-31T08:32:15.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1709864960 bytes 2024-08-31T08:32:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1711254528 bytes 2024-08-31T08:32:16.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1734625280 bytes 2024-08-31T08:32:16.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1791060992 bytes 2024-08-31T08:32:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1873912832 bytes 2024-08-31T08:32:17.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1980660736 bytes 2024-08-31T08:32:17.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1994549248 bytes 2024-08-31T08:32:17.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2027286528 bytes 2024-08-31T08:32:17.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2056496128 bytes 2024-08-31T08:32:17.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2117282816 bytes 2024-08-31T08:32:17.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2183144448 bytes 2024-08-31T08:32:18.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2203748352 bytes 2024-08-31T08:32:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2253076480 bytes 2024-08-31T08:32:18.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2299964416 bytes 2024-08-31T08:32:18.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2303222784 bytes 2024-08-31T08:32:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2349491200 bytes 2024-08-31T08:32:18.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2409913344 bytes 2024-08-31T08:32:19.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2492029952 bytes 2024-08-31T08:32:19.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2561605632 bytes 2024-08-31T08:32:19.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2609321984 bytes 2024-08-31T08:32:19.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2635283456 bytes 2024-08-31T08:32:19.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2664644608 bytes 2024-08-31T08:32:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2724916224 bytes 2024-08-31T08:32:20.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2749765632 bytes 2024-08-31T08:32:20.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2800863232 bytes 2024-08-31T08:32:20.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2824734720 bytes 2024-08-31T08:32:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:32:20.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_8 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:20.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_8 128M 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:20.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_8 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:20.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:32:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:20.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-08-31T08:32:21.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:32:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-08-31T08:32:21.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_8': 2024-08-31T08:32:21.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:32:21.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:32:21.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12c6f2d2a822 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12c6f2d2a822 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:20 2024 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:20 2024 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:20 2024 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 29308e73-15d8-45be-8eab-74c2808c5487 2024-08-31T08:32:21.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_8 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:32:21.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:32:21.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:32:21.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:32:21.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:32:21.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 59868160 bytes 2024-08-31T08:32:22.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 142376960 bytes 2024-08-31T08:32:22.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 201131008 bytes 2024-08-31T08:32:22.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 203891712 bytes 2024-08-31T08:32:22.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 215189504 bytes 2024-08-31T08:32:22.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 253507584 bytes 2024-08-31T08:32:22.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 303843328 bytes 2024-08-31T08:32:22.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 356147200 bytes 2024-08-31T08:32:23.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 425482240 bytes 2024-08-31T08:32:23.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 499817472 bytes 2024-08-31T08:32:23.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 522863616 bytes 2024-08-31T08:32:23.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 556155904 bytes 2024-08-31T08:32:23.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 614004736 bytes 2024-08-31T08:32:24.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 679970816 bytes 2024-08-31T08:32:24.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 765667328 bytes 2024-08-31T08:32:24.430 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 791200768 bytes 2024-08-31T08:32:24.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 815412224 bytes 2024-08-31T08:32:24.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 859354112 bytes 2024-08-31T08:32:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 916784128 bytes 2024-08-31T08:32:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 953926656 bytes 2024-08-31T08:32:25.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 987602944 bytes 2024-08-31T08:32:25.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1016298496 bytes 2024-08-31T08:32:25.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1034850304 bytes 2024-08-31T08:32:25.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1044360192 bytes 2024-08-31T08:32:25.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1147043840 bytes 2024-08-31T08:32:25.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1155055616 bytes 2024-08-31T08:32:26.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1197941760 bytes 2024-08-31T08:32:26.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1239702528 bytes 2024-08-31T08:32:26.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1256886272 bytes 2024-08-31T08:32:26.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1286956032 bytes 2024-08-31T08:32:26.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1334633472 bytes 2024-08-31T08:32:26.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1388882944 bytes 2024-08-31T08:32:27.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1422267392 bytes 2024-08-31T08:32:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1444910080 bytes 2024-08-31T08:32:27.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1469632512 bytes 2024-08-31T08:32:27.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1530585088 bytes 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_9 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_9 128M 2024-08-31T08:32:27.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:27.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_9 2024-08-31T08:32:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:32:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:32:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:27.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-08-31T08:32:28.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:32:28.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-08-31T08:32:28.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_9': 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12d241ddc5c8 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12d241ddc5c8 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:27 2024 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:27 2024 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:27 2024 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:32:28.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:32:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: a832ab25-9838-4884-8321-006c701241dd 2024-08-31T08:32:28.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:32:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:32:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_9 2024-08-31T08:32:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:32:28.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:32:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:32:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:32:28.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:32:28.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 40150016 bytes 2024-08-31T08:32:29.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 197136384 bytes 2024-08-31T08:32:29.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 197779456 bytes 2024-08-31T08:32:29.433 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 241447936 bytes 2024-08-31T08:32:29.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 269544448 bytes 2024-08-31T08:32:29.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 323679232 bytes 2024-08-31T08:32:29.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 347066368 bytes 2024-08-31T08:32:30.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 468738048 bytes 2024-08-31T08:32:30.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 475717632 bytes 2024-08-31T08:32:30.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 496796672 bytes 2024-08-31T08:32:30.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 504071168 bytes 2024-08-31T08:32:30.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 520771584 bytes 2024-08-31T08:32:30.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 573158400 bytes 2024-08-31T08:32:31.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 624409600 bytes 2024-08-31T08:32:31.142 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 662866944 bytes 2024-08-31T08:32:31.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 681888768 bytes 2024-08-31T08:32:31.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 731932672 bytes 2024-08-31T08:32:31.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 773227520 bytes 2024-08-31T08:32:31.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 798594048 bytes 2024-08-31T08:32:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 823110656 bytes 2024-08-31T08:32:31.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 823707648 bytes 2024-08-31T08:32:32.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 850169856 bytes 2024-08-31T08:32:32.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 891902976 bytes 2024-08-31T08:32:32.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 950278144 bytes 2024-08-31T08:32:32.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1019364352 bytes 2024-08-31T08:32:32.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1025772544 bytes 2024-08-31T08:32:32.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1052223488 bytes 2024-08-31T08:32:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1058387968 bytes 2024-08-31T08:32:33.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1100700672 bytes 2024-08-31T08:32:33.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1161902080 bytes 2024-08-31T08:32:33.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1186156544 bytes 2024-08-31T08:32:33.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1208017920 bytes 2024-08-31T08:32:33.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1254870016 bytes 2024-08-31T08:32:33.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1280161792 bytes 2024-08-31T08:32:33.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1315847168 bytes 2024-08-31T08:32:34.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1316656128 bytes 2024-08-31T08:32:34.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1374076928 bytes 2024-08-31T08:32:34.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1430484992 bytes 2024-08-31T08:32:34.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1458926592 bytes 2024-08-31T08:32:34.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1533296640 bytes 2024-08-31T08:32:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1558501376 bytes 2024-08-31T08:32:34.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1572185088 bytes 2024-08-31T08:32:35.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1598303232 bytes 2024-08-31T08:32:35.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1629716480 bytes 2024-08-31T08:32:35.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1686360064 bytes 2024-08-31T08:32:35.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1714160640 bytes 2024-08-31T08:32:35.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1782966272 bytes 2024-08-31T08:32:36.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1853923328 bytes 2024-08-31T08:32:36.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1891493888 bytes 2024-08-31T08:32:36.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1949318144 bytes 2024-08-31T08:32:36.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2012840960 bytes 2024-08-31T08:32:36.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2086944768 bytes 2024-08-31T08:32:36.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2115998720 bytes 2024-08-31T08:32:37.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2210847744 bytes 2024-08-31T08:32:37.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2211460096 bytes 2024-08-31T08:32:37.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2263991296 bytes 2024-08-31T08:32:37.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2264625152 bytes 2024-08-31T08:32:37.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2342998016 bytes 2024-08-31T08:32:38.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2411276288 bytes 2024-08-31T08:32:38.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2476989440 bytes 2024-08-31T08:32:38.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2584353792 bytes 2024-08-31T08:32:38.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2637452288 bytes 2024-08-31T08:32:38.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2734528512 bytes 2024-08-31T08:32:39.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2757412864 bytes 2024-08-31T08:32:39.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2842514432 bytes 2024-08-31T08:32:39.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2929076224 bytes 2024-08-31T08:32:39.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2975851520 bytes 2024-08-31T08:32:39.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 3023309824 bytes 2024-08-31T08:32:40.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 3094435840 bytes 2024-08-31T08:32:40.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3142860800 bytes 2024-08-31T08:32:40.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3199833088 bytes 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_10 2024-08-31T08:32:40.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_10 128M 2024-08-31T08:32:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:40.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_10 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:40.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-08-31T08:32:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:32:41.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-08-31T08:32:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_10': 2024-08-31T08:32:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:32:41.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12de93748154 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12de93748154 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:40 2024 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:40 2024 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:40 2024 2024-08-31T08:32:41.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:32:41.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:32:41.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 138c0f6a-f5c2-4054-b5be-480015cc0759 2024-08-31T08:32:41.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_10 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:32:41.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:32:41.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:32:41.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:32:41.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:32:41.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 35829760 bytes 2024-08-31T08:32:42.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 122363904 bytes 2024-08-31T08:32:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 136988672 bytes 2024-08-31T08:32:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 168422400 bytes 2024-08-31T08:32:42.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 243022848 bytes 2024-08-31T08:32:42.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 243625984 bytes 2024-08-31T08:32:42.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 250563584 bytes 2024-08-31T08:32:42.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 311203840 bytes 2024-08-31T08:32:43.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 336858112 bytes 2024-08-31T08:32:43.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 340410368 bytes 2024-08-31T08:32:43.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 366440448 bytes 2024-08-31T08:32:43.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 456788992 bytes 2024-08-31T08:32:43.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 492892160 bytes 2024-08-31T08:32:43.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 553170944 bytes 2024-08-31T08:32:44.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 561193984 bytes 2024-08-31T08:32:44.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 595829760 bytes 2024-08-31T08:32:44.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 621104128 bytes 2024-08-31T08:32:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 679241728 bytes 2024-08-31T08:32:44.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 716180480 bytes 2024-08-31T08:32:44.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 749331456 bytes 2024-08-31T08:32:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 794469376 bytes 2024-08-31T08:32:45.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 803939328 bytes 2024-08-31T08:32:45.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 887708672 bytes 2024-08-31T08:32:45.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 917698560 bytes 2024-08-31T08:32:45.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 946185216 bytes 2024-08-31T08:32:45.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 988466176 bytes 2024-08-31T08:32:45.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1034056704 bytes 2024-08-31T08:32:46.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1050955776 bytes 2024-08-31T08:32:46.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1058872320 bytes 2024-08-31T08:32:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1179574272 bytes 2024-08-31T08:32:46.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1229039616 bytes 2024-08-31T08:32:46.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1255430144 bytes 2024-08-31T08:32:46.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1300267008 bytes 2024-08-31T08:32:47.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1384918016 bytes 2024-08-31T08:32:47.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1397961728 bytes 2024-08-31T08:32:47.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1411403776 bytes 2024-08-31T08:32:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1431535616 bytes 2024-08-31T08:32:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:32:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:32:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:32:47.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_11 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_11 128M 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:32:47.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_11 2024-08-31T08:32:47.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:32:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2024-08-31T08:32:48.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:32:48.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_11': 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12eb9caeff2d 2024-08-31T08:32:48.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12eb9caeff2d 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:47 2024 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:47 2024 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:47 2024 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 543fba89-43e8-4a84-a10b-ed65255deb74 2024-08-31T08:32:48.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:32:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:32:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_11 2024-08-31T08:32:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:32:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:32:48.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:32:48.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:32:48.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:32:48.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:32:48.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:32:48.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 615424 bytes 2024-08-31T08:32:49.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 84207616 bytes 2024-08-31T08:32:49.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 105031680 bytes 2024-08-31T08:32:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 158262272 bytes 2024-08-31T08:32:49.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 191984640 bytes 2024-08-31T08:32:49.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 192616448 bytes 2024-08-31T08:32:49.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 227310592 bytes 2024-08-31T08:32:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 307363840 bytes 2024-08-31T08:32:50.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 395321344 bytes 2024-08-31T08:32:50.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4348642131196885) > weight (0.25) 2024-08-31T08:32:50.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3887635725892179) > weight (0.25) 2024-08-31T08:32:50.336 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9258676835381201) > weight (0.25) 2024-08-31T08:32:50.337 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.711101902085578) > weight (0.25) 2024-08-31T08:32:50.337 INFO:tasks.rbd_mirror_thrash:waiting for 73 secs before thrashing 2024-08-31T08:32:50.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 422670336 bytes 2024-08-31T08:32:50.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 465202176 bytes 2024-08-31T08:32:50.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 522464256 bytes 2024-08-31T08:32:50.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 542934016 bytes 2024-08-31T08:32:51.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 596511744 bytes 2024-08-31T08:32:51.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 643760128 bytes 2024-08-31T08:32:51.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 658782208 bytes 2024-08-31T08:32:51.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 700228608 bytes 2024-08-31T08:32:51.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 742866944 bytes 2024-08-31T08:32:52.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 820195328 bytes 2024-08-31T08:32:52.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 858057728 bytes 2024-08-31T08:32:52.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 897628160 bytes 2024-08-31T08:32:52.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 949737472 bytes 2024-08-31T08:32:52.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1010906112 bytes 2024-08-31T08:32:52.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1042179072 bytes 2024-08-31T08:32:52.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1056129024 bytes 2024-08-31T08:32:53.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1157397504 bytes 2024-08-31T08:32:53.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1175490560 bytes 2024-08-31T08:32:53.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1200843776 bytes 2024-08-31T08:32:53.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1201465344 bytes 2024-08-31T08:32:53.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1293694976 bytes 2024-08-31T08:32:54.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1374438400 bytes 2024-08-31T08:32:54.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1391532032 bytes 2024-08-31T08:32:54.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1453554688 bytes 2024-08-31T08:32:54.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1471267840 bytes 2024-08-31T08:32:54.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1507241984 bytes 2024-08-31T08:32:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1548654592 bytes 2024-08-31T08:32:54.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1589868544 bytes 2024-08-31T08:32:55.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1623406592 bytes 2024-08-31T08:32:55.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1660725248 bytes 2024-08-31T08:32:55.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1691310080 bytes 2024-08-31T08:32:55.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1718122496 bytes 2024-08-31T08:32:55.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1761226752 bytes 2024-08-31T08:32:55.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1816840192 bytes 2024-08-31T08:32:56.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1854640128 bytes 2024-08-31T08:32:56.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1915663360 bytes 2024-08-31T08:32:56.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1945078784 bytes 2024-08-31T08:32:56.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1974434816 bytes 2024-08-31T08:32:56.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1982638080 bytes 2024-08-31T08:32:56.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1995018240 bytes 2024-08-31T08:32:56.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2060081152 bytes 2024-08-31T08:32:57.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2062511104 bytes 2024-08-31T08:32:57.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2076084224 bytes 2024-08-31T08:32:57.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2078531584 bytes 2024-08-31T08:32:57.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2079137792 bytes 2024-08-31T08:32:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2150327296 bytes 2024-08-31T08:32:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2212583424 bytes 2024-08-31T08:32:57.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2251288576 bytes 2024-08-31T08:32:58.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2274357248 bytes 2024-08-31T08:32:58.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2280074240 bytes 2024-08-31T08:32:58.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2297358336 bytes 2024-08-31T08:32:58.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2365437952 bytes 2024-08-31T08:32:58.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2386415616 bytes 2024-08-31T08:32:58.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2447866880 bytes 2024-08-31T08:32:59.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2508240896 bytes 2024-08-31T08:32:59.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2563207168 bytes 2024-08-31T08:32:59.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2590130176 bytes 2024-08-31T08:32:59.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2622889984 bytes 2024-08-31T08:32:59.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2696304640 bytes 2024-08-31T08:32:59.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2759425024 bytes 2024-08-31T08:33:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2832970752 bytes 2024-08-31T08:33:00.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2922318848 bytes 2024-08-31T08:33:00.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2937635840 bytes 2024-08-31T08:33:00.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 2995324928 bytes 2024-08-31T08:33:01.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3007867904 bytes 2024-08-31T08:33:01.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3049692160 bytes 2024-08-31T08:33:01.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3063613440 bytes 2024-08-31T08:33:01.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3094279168 bytes 2024-08-31T08:33:01.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3152851968 bytes 2024-08-31T08:33:01.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_12 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_12 128M 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:01.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:01.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_12 2024-08-31T08:33:01.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:01.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:01.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2024-08-31T08:33:02.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:02.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2024-08-31T08:33:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_12': 2024-08-31T08:33:02.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 12f65b8cf4d3 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.12f65b8cf4d3 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:01 2024 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:01 2024 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:01 2024 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:02.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 32719a8b-c4e0-48d1-af14-d5d096fbbdaa 2024-08-31T08:33:02.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_12 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:33:02.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:02.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=14 2024-08-31T08:33:02.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:02.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:02.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 610304 bytes 2024-08-31T08:33:03.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 64028672 bytes 2024-08-31T08:33:03.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 112125952 bytes 2024-08-31T08:33:03.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 140384256 bytes 2024-08-31T08:33:03.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 231289856 bytes 2024-08-31T08:33:03.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 290945024 bytes 2024-08-31T08:33:04.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 319038464 bytes 2024-08-31T08:33:04.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 341456896 bytes 2024-08-31T08:33:05.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 414278656 bytes 2024-08-31T08:33:05.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 428380160 bytes 2024-08-31T08:33:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 467434496 bytes 2024-08-31T08:33:05.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 516349952 bytes 2024-08-31T08:33:05.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 530497536 bytes 2024-08-31T08:33:05.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 560914432 bytes 2024-08-31T08:33:05.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 576059392 bytes 2024-08-31T08:33:06.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 621582336 bytes 2024-08-31T08:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 632230912 bytes 2024-08-31T08:33:06.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 653069312 bytes 2024-08-31T08:33:06.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 724143104 bytes 2024-08-31T08:33:06.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 788040704 bytes 2024-08-31T08:33:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 814588928 bytes 2024-08-31T08:33:06.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 846433280 bytes 2024-08-31T08:33:07.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 901377024 bytes 2024-08-31T08:33:07.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 993816576 bytes 2024-08-31T08:33:07.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1055296512 bytes 2024-08-31T08:33:07.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1107309568 bytes 2024-08-31T08:33:07.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1159055360 bytes 2024-08-31T08:33:08.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1190751232 bytes 2024-08-31T08:33:08.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1198896128 bytes 2024-08-31T08:33:08.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1213521920 bytes 2024-08-31T08:33:08.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1227273216 bytes 2024-08-31T08:33:08.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1253214208 bytes 2024-08-31T08:33:08.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1254906880 bytes 2024-08-31T08:33:08.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1327960064 bytes 2024-08-31T08:33:08.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1370252288 bytes 2024-08-31T08:33:09.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1378334720 bytes 2024-08-31T08:33:09.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1432285184 bytes 2024-08-31T08:33:09.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1458221056 bytes 2024-08-31T08:33:09.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1517084672 bytes 2024-08-31T08:33:09.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1564267520 bytes 2024-08-31T08:33:09.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1591766016 bytes 2024-08-31T08:33:10.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1636066304 bytes 2024-08-31T08:33:10.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1706398720 bytes 2024-08-31T08:33:10.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1824045056 bytes 2024-08-31T08:33:10.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1901329408 bytes 2024-08-31T08:33:10.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1960629248 bytes 2024-08-31T08:33:11.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1967853568 bytes 2024-08-31T08:33:11.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1980263424 bytes 2024-08-31T08:33:11.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1993624576 bytes 2024-08-31T08:33:11.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2101216256 bytes 2024-08-31T08:33:11.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2142458880 bytes 2024-08-31T08:33:12.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2211822592 bytes 2024-08-31T08:33:12.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2236146688 bytes 2024-08-31T08:33:12.242 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5281332670916398) > weight (0.25) 2024-08-31T08:33:12.242 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6899020095434294) > weight (0.25) 2024-08-31T08:33:12.242 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.390514906382745) > weight (0.25) 2024-08-31T08:33:12.242 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6130150619066109) > weight (0.25) 2024-08-31T08:33:12.242 INFO:tasks.rbd_mirror_thrash:waiting for 85 secs before thrashing 2024-08-31T08:33:12.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2252266496 bytes 2024-08-31T08:33:12.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2286767104 bytes 2024-08-31T08:33:12.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2350659584 bytes 2024-08-31T08:33:12.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2371275776 bytes 2024-08-31T08:33:12.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2407624704 bytes 2024-08-31T08:33:13.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2556575744 bytes 2024-08-31T08:33:13.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2573616128 bytes 2024-08-31T08:33:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2644171776 bytes 2024-08-31T08:33:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2669640704 bytes 2024-08-31T08:33:13.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2744981504 bytes 2024-08-31T08:33:14.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2756367360 bytes 2024-08-31T08:33:14.250 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2790846464 bytes 2024-08-31T08:33:14.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2842085376 bytes 2024-08-31T08:33:14.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2889287680 bytes 2024-08-31T08:33:14.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2898342912 bytes 2024-08-31T08:33:15.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2935063552 bytes 2024-08-31T08:33:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3068824576 bytes 2024-08-31T08:33:15.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3108219904 bytes 2024-08-31T08:33:15.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3158779904 bytes 2024-08-31T08:33:15.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3235810304 bytes 2024-08-31T08:33:16.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3282052096 bytes 2024-08-31T08:33:16.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3319280640 bytes 2024-08-31T08:33:16.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3408910336 bytes 2024-08-31T08:33:16.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3432200192 bytes 2024-08-31T08:33:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3476121600 bytes 2024-08-31T08:33:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_13 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_13 128M 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:16.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_13 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:33:16.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:16.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2024-08-31T08:33:17.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:17.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_13': 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13028e013f7f 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13028e013f7f 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:16 2024 2024-08-31T08:33:17.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:16 2024 2024-08-31T08:33:17.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:16 2024 2024-08-31T08:33:17.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:17.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:17.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 942a444b-45e7-45c0-b6f0-0f1f234cac1e 2024-08-31T08:33:17.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:17.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:17.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_13 2024-08-31T08:33:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:33:17.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=8 2024-08-31T08:33:17.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:17.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:17.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 17747968 bytes 2024-08-31T08:33:17.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 25012224 bytes 2024-08-31T08:33:18.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 57442304 bytes 2024-08-31T08:33:18.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 79691776 bytes 2024-08-31T08:33:18.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 116997120 bytes 2024-08-31T08:33:18.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 177950720 bytes 2024-08-31T08:33:18.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 234644480 bytes 2024-08-31T08:33:18.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 281858048 bytes 2024-08-31T08:33:19.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 318980096 bytes 2024-08-31T08:33:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 328087552 bytes 2024-08-31T08:33:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 376559616 bytes 2024-08-31T08:33:19.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 446123008 bytes 2024-08-31T08:33:19.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 480220160 bytes 2024-08-31T08:33:20.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 534351872 bytes 2024-08-31T08:33:20.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 566852608 bytes 2024-08-31T08:33:20.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 586291200 bytes 2024-08-31T08:33:20.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 649069568 bytes 2024-08-31T08:33:20.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 649905152 bytes 2024-08-31T08:33:20.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 727143424 bytes 2024-08-31T08:33:21.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 793223168 bytes 2024-08-31T08:33:21.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 793815040 bytes 2024-08-31T08:33:21.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 889736192 bytes 2024-08-31T08:33:21.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 890352640 bytes 2024-08-31T08:33:21.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 924741632 bytes 2024-08-31T08:33:21.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 998457344 bytes 2024-08-31T08:33:21.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1010812928 bytes 2024-08-31T08:33:22.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1035672576 bytes 2024-08-31T08:33:22.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1131000832 bytes 2024-08-31T08:33:22.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1175957504 bytes 2024-08-31T08:33:22.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1219966976 bytes 2024-08-31T08:33:22.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1221738496 bytes 2024-08-31T08:33:22.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1237761024 bytes 2024-08-31T08:33:23.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1264762880 bytes 2024-08-31T08:33:23.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1312357376 bytes 2024-08-31T08:33:23.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1351226368 bytes 2024-08-31T08:33:23.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1382496256 bytes 2024-08-31T08:33:23.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1421315072 bytes 2024-08-31T08:33:23.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1447011328 bytes 2024-08-31T08:33:23.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1495463936 bytes 2024-08-31T08:33:24.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1530755072 bytes 2024-08-31T08:33:24.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1539235840 bytes 2024-08-31T08:33:24.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1568308224 bytes 2024-08-31T08:33:24.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1608400896 bytes 2024-08-31T08:33:24.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1609005056 bytes 2024-08-31T08:33:24.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1633275904 bytes 2024-08-31T08:33:24.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1664650240 bytes 2024-08-31T08:33:25.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1688924160 bytes 2024-08-31T08:33:25.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1706528768 bytes 2024-08-31T08:33:25.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1760307200 bytes 2024-08-31T08:33:25.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1851800576 bytes 2024-08-31T08:33:25.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:25.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:25.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_14 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_14 128M 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:25.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_14 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:25.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:25.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:33:25.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:25.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2024-08-31T08:33:26.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:26.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_14': 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 130eb039ff8c 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.130eb039ff8c 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:26.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:25 2024 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:25 2024 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:25 2024 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1ef17613-0e21-45d7-b5d2-ab9520155a3c 2024-08-31T08:33:26.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_14 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:33:26.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:33:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:26.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:26.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 936960 bytes 2024-08-31T08:33:26.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 18590720 bytes 2024-08-31T08:33:27.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 42574848 bytes 2024-08-31T08:33:27.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 131564544 bytes 2024-08-31T08:33:27.430 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 144184320 bytes 2024-08-31T08:33:27.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 251523072 bytes 2024-08-31T08:33:27.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 278437888 bytes 2024-08-31T08:33:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 338032640 bytes 2024-08-31T08:33:28.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 374949888 bytes 2024-08-31T08:33:28.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 394810368 bytes 2024-08-31T08:33:28.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 428727296 bytes 2024-08-31T08:33:28.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 463062016 bytes 2024-08-31T08:33:28.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 463866880 bytes 2024-08-31T08:33:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 505952256 bytes 2024-08-31T08:33:29.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 534827008 bytes 2024-08-31T08:33:29.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 548024320 bytes 2024-08-31T08:33:29.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 585740288 bytes 2024-08-31T08:33:29.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 642409472 bytes 2024-08-31T08:33:29.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 725752832 bytes 2024-08-31T08:33:29.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 802841600 bytes 2024-08-31T08:33:30.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 810699776 bytes 2024-08-31T08:33:30.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 864819200 bytes 2024-08-31T08:33:30.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 927809536 bytes 2024-08-31T08:33:30.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 965652480 bytes 2024-08-31T08:33:30.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 998100992 bytes 2024-08-31T08:33:31.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1081067520 bytes 2024-08-31T08:33:31.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1123834880 bytes 2024-08-31T08:33:31.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1159407616 bytes 2024-08-31T08:33:31.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1188434944 bytes 2024-08-31T08:33:31.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1231482880 bytes 2024-08-31T08:33:31.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1252214784 bytes 2024-08-31T08:33:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1295188992 bytes 2024-08-31T08:33:32.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1361021952 bytes 2024-08-31T08:33:32.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1404593152 bytes 2024-08-31T08:33:32.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1450379264 bytes 2024-08-31T08:33:32.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1457230848 bytes 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_15 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2024-08-31T08:33:32.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_15 128M 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:32.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_15 2024-08-31T08:33:32.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:32.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2024-08-31T08:33:33.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:33.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2024-08-31T08:33:33.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_15': 2024-08-31T08:33:33.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:33.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:33.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 131a1902a195 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.131a1902a195 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:32 2024 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:32 2024 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:32 2024 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: ff694f40-7359-4371-9cac-7bf8444def68 2024-08-31T08:33:33.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_15 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:33:33.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:33.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=8 2024-08-31T08:33:33.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:33.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:34.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 74399744 bytes 2024-08-31T08:33:34.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 116506624 bytes 2024-08-31T08:33:34.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 147227648 bytes 2024-08-31T08:33:34.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 166341632 bytes 2024-08-31T08:33:34.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 181155840 bytes 2024-08-31T08:33:34.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 194197504 bytes 2024-08-31T08:33:34.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 221661184 bytes 2024-08-31T08:33:34.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 240595968 bytes 2024-08-31T08:33:35.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 275808256 bytes 2024-08-31T08:33:35.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 359242752 bytes 2024-08-31T08:33:35.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 379506688 bytes 2024-08-31T08:33:35.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 415782912 bytes 2024-08-31T08:33:35.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 501856256 bytes 2024-08-31T08:33:36.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 562593792 bytes 2024-08-31T08:33:36.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 636484608 bytes 2024-08-31T08:33:36.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 722391040 bytes 2024-08-31T08:33:36.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 779220992 bytes 2024-08-31T08:33:36.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 809428992 bytes 2024-08-31T08:33:37.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 810016768 bytes 2024-08-31T08:33:37.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 836447232 bytes 2024-08-31T08:33:37.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 864898048 bytes 2024-08-31T08:33:37.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 970212352 bytes 2024-08-31T08:33:37.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 982019072 bytes 2024-08-31T08:33:37.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 991996928 bytes 2024-08-31T08:33:38.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1065112576 bytes 2024-08-31T08:33:38.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1134529536 bytes 2024-08-31T08:33:38.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1189200896 bytes 2024-08-31T08:33:38.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1251333120 bytes 2024-08-31T08:33:38.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1305061376 bytes 2024-08-31T08:33:39.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1402778624 bytes 2024-08-31T08:33:39.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1444720640 bytes 2024-08-31T08:33:39.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1485081600 bytes 2024-08-31T08:33:39.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1526099968 bytes 2024-08-31T08:33:39.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1562798080 bytes 2024-08-31T08:33:39.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1581137920 bytes 2024-08-31T08:33:40.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1646315520 bytes 2024-08-31T08:33:40.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1669462016 bytes 2024-08-31T08:33:40.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1707844608 bytes 2024-08-31T08:33:40.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1762492416 bytes 2024-08-31T08:33:40.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1827474432 bytes 2024-08-31T08:33:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1846077440 bytes 2024-08-31T08:33:41.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1861987328 bytes 2024-08-31T08:33:41.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1883395072 bytes 2024-08-31T08:33:41.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1938744320 bytes 2024-08-31T08:33:41.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1972303872 bytes 2024-08-31T08:33:41.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2013713408 bytes 2024-08-31T08:33:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:41.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_16 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_16 128M 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_16 2024-08-31T08:33:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2024-08-31T08:33:42.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2024-08-31T08:33:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_16': 2024-08-31T08:33:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:42.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1326f24475fe 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1326f24475fe 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:41 2024 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:41 2024 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:41 2024 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 09ffa3ae-165e-4e02-b27b-6c2e5c0d8dff 2024-08-31T08:33:42.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_16 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:33:42.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:33:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:43.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 29813760 bytes 2024-08-31T08:33:43.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 79681536 bytes 2024-08-31T08:33:43.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 113212416 bytes 2024-08-31T08:33:43.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 143227904 bytes 2024-08-31T08:33:43.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 161677312 bytes 2024-08-31T08:33:43.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 177947648 bytes 2024-08-31T08:33:43.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 199299072 bytes 2024-08-31T08:33:44.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 244385792 bytes 2024-08-31T08:33:44.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 257665024 bytes 2024-08-31T08:33:44.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 259101696 bytes 2024-08-31T08:33:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 309200896 bytes 2024-08-31T08:33:44.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 386524160 bytes 2024-08-31T08:33:44.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 436467712 bytes 2024-08-31T08:33:44.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 464161792 bytes 2024-08-31T08:33:45.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 467504128 bytes 2024-08-31T08:33:45.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 531810304 bytes 2024-08-31T08:33:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 585951232 bytes 2024-08-31T08:33:45.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 620461056 bytes 2024-08-31T08:33:45.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 621029376 bytes 2024-08-31T08:33:45.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 639616000 bytes 2024-08-31T08:33:45.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 650660864 bytes 2024-08-31T08:33:45.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 653199360 bytes 2024-08-31T08:33:46.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 692156416 bytes 2024-08-31T08:33:46.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 702991360 bytes 2024-08-31T08:33:46.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 709809152 bytes 2024-08-31T08:33:46.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 710439936 bytes 2024-08-31T08:33:46.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 755212288 bytes 2024-08-31T08:33:46.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 778665984 bytes 2024-08-31T08:33:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 819223552 bytes 2024-08-31T08:33:47.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 851419136 bytes 2024-08-31T08:33:47.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 904747008 bytes 2024-08-31T08:33:47.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 989027328 bytes 2024-08-31T08:33:47.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1033124864 bytes 2024-08-31T08:33:47.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1058413568 bytes 2024-08-31T08:33:47.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1096681472 bytes 2024-08-31T08:33:48.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1151618048 bytes 2024-08-31T08:33:48.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1179058176 bytes 2024-08-31T08:33:48.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1210902528 bytes 2024-08-31T08:33:48.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1223419904 bytes 2024-08-31T08:33:48.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1232845824 bytes 2024-08-31T08:33:48.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1310146560 bytes 2024-08-31T08:33:49.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1343344640 bytes 2024-08-31T08:33:49.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1398513664 bytes 2024-08-31T08:33:49.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1473282048 bytes 2024-08-31T08:33:49.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1570500608 bytes 2024-08-31T08:33:49.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1603172352 bytes 2024-08-31T08:33:50.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1664529408 bytes 2024-08-31T08:33:50.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1676617728 bytes 2024-08-31T08:33:50.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1702206464 bytes 2024-08-31T08:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1774239744 bytes 2024-08-31T08:33:50.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1790918656 bytes 2024-08-31T08:33:51.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 1892334592 bytes 2024-08-31T08:33:51.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 1963981824 bytes 2024-08-31T08:33:51.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2032037888 bytes 2024-08-31T08:33:51.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2057260032 bytes 2024-08-31T08:33:51.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2086642688 bytes 2024-08-31T08:33:51.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2115057664 bytes 2024-08-31T08:33:52.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2167343104 bytes 2024-08-31T08:33:52.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2221203456 bytes 2024-08-31T08:33:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2281526272 bytes 2024-08-31T08:33:52.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2343454720 bytes 2024-08-31T08:33:52.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2405644288 bytes 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_17 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2024-08-31T08:33:52.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_17 128M 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:33:52.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_17 2024-08-31T08:33:52.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:33:52.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:33:52.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2024-08-31T08:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:33:53.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_17': 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 133231d6cab8 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.133231d6cab8 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:52 2024 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:52 2024 2024-08-31T08:33:53.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:52 2024 2024-08-31T08:33:53.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:33:53.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:33:53.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 188a5b44-125b-4376-a8fb-33c918538c19 2024-08-31T08:33:53.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_17 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:33:53.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:33:53.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:33:53.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:33:53.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:33:53.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 589824 bytes 2024-08-31T08:33:53.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 46078976 bytes 2024-08-31T08:33:54.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 98834432 bytes 2024-08-31T08:33:54.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 104283136 bytes 2024-08-31T08:33:54.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 143128576 bytes 2024-08-31T08:33:54.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 180616192 bytes 2024-08-31T08:33:54.780 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 193759232 bytes 2024-08-31T08:33:55.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 259454976 bytes 2024-08-31T08:33:55.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 282372096 bytes 2024-08-31T08:33:55.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 313907200 bytes 2024-08-31T08:33:55.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 345036800 bytes 2024-08-31T08:33:55.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 369545216 bytes 2024-08-31T08:33:55.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 425431040 bytes 2024-08-31T08:33:55.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 426047488 bytes 2024-08-31T08:33:56.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 525236224 bytes 2024-08-31T08:33:56.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 530054144 bytes 2024-08-31T08:33:56.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 556562432 bytes 2024-08-31T08:33:56.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 643765248 bytes 2024-08-31T08:33:56.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 685733888 bytes 2024-08-31T08:33:56.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 697118720 bytes 2024-08-31T08:33:57.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 787983360 bytes 2024-08-31T08:33:57.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 814556160 bytes 2024-08-31T08:33:57.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 863217664 bytes 2024-08-31T08:33:57.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 865793024 bytes 2024-08-31T08:33:57.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 893505536 bytes 2024-08-31T08:33:57.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 911632384 bytes 2024-08-31T08:33:58.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 968825856 bytes 2024-08-31T08:33:58.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 970379264 bytes 2024-08-31T08:33:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1007879168 bytes 2024-08-31T08:33:58.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1034853376 bytes 2024-08-31T08:33:58.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1105994752 bytes 2024-08-31T08:33:58.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1159003136 bytes 2024-08-31T08:33:59.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1202482176 bytes 2024-08-31T08:33:59.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1282973696 bytes 2024-08-31T08:33:59.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1317937152 bytes 2024-08-31T08:33:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1370362880 bytes 2024-08-31T08:33:59.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1420900352 bytes 2024-08-31T08:33:59.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1468471296 bytes 2024-08-31T08:34:00.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1509852160 bytes 2024-08-31T08:34:00.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1559816192 bytes 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_18 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2024-08-31T08:34:00.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_18 128M 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:00.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_18 2024-08-31T08:34:00.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:34:00.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:00.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:00.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2024-08-31T08:34:01.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:34:01.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_18': 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 133e6575200b 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.133e6575200b 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:34:01.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:00 2024 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:00 2024 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:00 2024 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 9210fc52-db1b-4553-ad9c-5982d4188f4f 2024-08-31T08:34:01.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:34:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:34:01.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_18 2024-08-31T08:34:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:34:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:34:01.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:34:01.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:34:01.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:34:01.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 49987584 bytes 2024-08-31T08:34:01.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 62923776 bytes 2024-08-31T08:34:02.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 130072576 bytes 2024-08-31T08:34:02.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 148371456 bytes 2024-08-31T08:34:02.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 166929408 bytes 2024-08-31T08:34:02.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 199704576 bytes 2024-08-31T08:34:02.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 248884224 bytes 2024-08-31T08:34:02.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 352045056 bytes 2024-08-31T08:34:03.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 367028224 bytes 2024-08-31T08:34:03.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 395464704 bytes 2024-08-31T08:34:03.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 396883968 bytes 2024-08-31T08:34:03.338 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6698016144907113) > weight (0.25) 2024-08-31T08:34:03.338 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.958511673872124) > weight (0.25) 2024-08-31T08:34:03.338 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.303654018515663) > weight (0.25) 2024-08-31T08:34:03.338 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7763022315173218) > weight (0.25) 2024-08-31T08:34:03.338 INFO:tasks.rbd_mirror_thrash:waiting for 75 secs before thrashing 2024-08-31T08:34:03.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 426603520 bytes 2024-08-31T08:34:03.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 428147712 bytes 2024-08-31T08:34:03.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 458589184 bytes 2024-08-31T08:34:03.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 530928640 bytes 2024-08-31T08:34:04.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 553405440 bytes 2024-08-31T08:34:04.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 588082176 bytes 2024-08-31T08:34:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 642804736 bytes 2024-08-31T08:34:04.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 720165888 bytes 2024-08-31T08:34:04.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 747503616 bytes 2024-08-31T08:34:04.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 768899072 bytes 2024-08-31T08:34:05.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 837525504 bytes 2024-08-31T08:34:05.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 912622592 bytes 2024-08-31T08:34:05.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 964507648 bytes 2024-08-31T08:34:05.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 997728256 bytes 2024-08-31T08:34:05.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1043292160 bytes 2024-08-31T08:34:06.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1063565312 bytes 2024-08-31T08:34:06.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1124428800 bytes 2024-08-31T08:34:06.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1174611968 bytes 2024-08-31T08:34:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1207453696 bytes 2024-08-31T08:34:06.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1254498304 bytes 2024-08-31T08:34:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1334296576 bytes 2024-08-31T08:34:07.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1344591872 bytes 2024-08-31T08:34:07.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1372996608 bytes 2024-08-31T08:34:07.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1423707136 bytes 2024-08-31T08:34:07.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1450642432 bytes 2024-08-31T08:34:07.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:34:07.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:34:07.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:34:07.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_19 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:34:07.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_19 128M 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:07.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_19 2024-08-31T08:34:07.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:34:07.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:07.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:34:07.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:07.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2024-08-31T08:34:08.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:34:08.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2024-08-31T08:34:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_19': 2024-08-31T08:34:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:34:08.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 134acc6f5cc9 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.134acc6f5cc9 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:07 2024 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:07 2024 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:07 2024 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:34:08.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d6dd38ac-296d-4d45-a399-c21cb2856fd7 2024-08-31T08:34:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_19 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:34:08.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:34:08.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:34:08.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:34:08.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:34:08.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 41925632 bytes 2024-08-31T08:34:08.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 46691328 bytes 2024-08-31T08:34:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 61536256 bytes 2024-08-31T08:34:09.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 112696320 bytes 2024-08-31T08:34:09.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 129417216 bytes 2024-08-31T08:34:09.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 130048000 bytes 2024-08-31T08:34:09.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 153551872 bytes 2024-08-31T08:34:09.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 193055744 bytes 2024-08-31T08:34:09.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 222862336 bytes 2024-08-31T08:34:09.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 236285952 bytes 2024-08-31T08:34:10.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 317369344 bytes 2024-08-31T08:34:10.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 326823936 bytes 2024-08-31T08:34:10.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 350763008 bytes 2024-08-31T08:34:10.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 376776704 bytes 2024-08-31T08:34:10.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 439021568 bytes 2024-08-31T08:34:11.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 546702336 bytes 2024-08-31T08:34:11.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 583496704 bytes 2024-08-31T08:34:11.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 668775424 bytes 2024-08-31T08:34:11.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 684514304 bytes 2024-08-31T08:34:11.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 704026624 bytes 2024-08-31T08:34:11.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 737403904 bytes 2024-08-31T08:34:12.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 823717888 bytes 2024-08-31T08:34:12.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 830225408 bytes 2024-08-31T08:34:12.379 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 834761728 bytes 2024-08-31T08:34:12.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 866842624 bytes 2024-08-31T08:34:12.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 938746880 bytes 2024-08-31T08:34:12.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 967379968 bytes 2024-08-31T08:34:13.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1042111488 bytes 2024-08-31T08:34:13.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1071057920 bytes 2024-08-31T08:34:13.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1094114304 bytes 2024-08-31T08:34:13.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1100401664 bytes 2024-08-31T08:34:13.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1168498688 bytes 2024-08-31T08:34:13.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1254982656 bytes 2024-08-31T08:34:14.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1308248064 bytes 2024-08-31T08:34:14.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1312425984 bytes 2024-08-31T08:34:14.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1371323392 bytes 2024-08-31T08:34:14.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1401924608 bytes 2024-08-31T08:34:14.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1487315968 bytes 2024-08-31T08:34:15.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1572400128 bytes 2024-08-31T08:34:15.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1658961920 bytes 2024-08-31T08:34:15.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1723394048 bytes 2024-08-31T08:34:15.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1781094400 bytes 2024-08-31T08:34:15.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1791380480 bytes 2024-08-31T08:34:16.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1817381888 bytes 2024-08-31T08:34:16.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1907942400 bytes 2024-08-31T08:34:16.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1973432320 bytes 2024-08-31T08:34:16.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1997283328 bytes 2024-08-31T08:34:16.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2046564352 bytes 2024-08-31T08:34:17.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2117822464 bytes 2024-08-31T08:34:17.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2182931456 bytes 2024-08-31T08:34:17.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2224577536 bytes 2024-08-31T08:34:17.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2249097216 bytes 2024-08-31T08:34:17.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2289174528 bytes 2024-08-31T08:34:17.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2309693440 bytes 2024-08-31T08:34:17.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2310312960 bytes 2024-08-31T08:34:18.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2341941248 bytes 2024-08-31T08:34:18.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2384532480 bytes 2024-08-31T08:34:18.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2434779136 bytes 2024-08-31T08:34:18.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2490554368 bytes 2024-08-31T08:34:18.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2511120384 bytes 2024-08-31T08:34:18.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2566666240 bytes 2024-08-31T08:34:19.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2674376704 bytes 2024-08-31T08:34:19.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2715820032 bytes 2024-08-31T08:34:19.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2771973120 bytes 2024-08-31T08:34:19.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2807096320 bytes 2024-08-31T08:34:20.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2842727424 bytes 2024-08-31T08:34:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2888855552 bytes 2024-08-31T08:34:20.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2940897280 bytes 2024-08-31T08:34:20.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2960196608 bytes 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_20 2024-08-31T08:34:20.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_20 128M 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:20.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_20 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:20.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:34:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2024-08-31T08:34:21.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:34:21.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2024-08-31T08:34:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_20': 2024-08-31T08:34:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:34:21.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1356e8ecdbe1 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1356e8ecdbe1 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:20 2024 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:20 2024 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:20 2024 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 221ab01e-6802-4692-8a08-3ac552ff07a3 2024-08-31T08:34:21.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:34:21.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:34:21.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_20 2024-08-31T08:34:21.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:21.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:21.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:34:21.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:34:21.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:34:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:34:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:34:21.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 7024640 bytes 2024-08-31T08:34:21.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 57886720 bytes 2024-08-31T08:34:22.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 97636352 bytes 2024-08-31T08:34:22.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 177519616 bytes 2024-08-31T08:34:22.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 286994432 bytes 2024-08-31T08:34:22.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 300936192 bytes 2024-08-31T08:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 327464960 bytes 2024-08-31T08:34:23.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 364467200 bytes 2024-08-31T08:34:23.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 387980288 bytes 2024-08-31T08:34:23.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 418642944 bytes 2024-08-31T08:34:23.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 440915968 bytes 2024-08-31T08:34:23.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 490037248 bytes 2024-08-31T08:34:23.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 529007616 bytes 2024-08-31T08:34:24.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 561472512 bytes 2024-08-31T08:34:24.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 571471872 bytes 2024-08-31T08:34:24.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 601260032 bytes 2024-08-31T08:34:24.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 626177024 bytes 2024-08-31T08:34:24.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 687213568 bytes 2024-08-31T08:34:24.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 735348736 bytes 2024-08-31T08:34:25.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 831353856 bytes 2024-08-31T08:34:25.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 856864768 bytes 2024-08-31T08:34:25.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 863311872 bytes 2024-08-31T08:34:25.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 938818560 bytes 2024-08-31T08:34:25.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 954684416 bytes 2024-08-31T08:34:25.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1008679936 bytes 2024-08-31T08:34:26.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1057868800 bytes 2024-08-31T08:34:26.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1153479680 bytes 2024-08-31T08:34:26.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1176222720 bytes 2024-08-31T08:34:26.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1195697152 bytes 2024-08-31T08:34:26.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1268358144 bytes 2024-08-31T08:34:27.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1325868032 bytes 2024-08-31T08:34:27.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1364153344 bytes 2024-08-31T08:34:27.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1415926784 bytes 2024-08-31T08:34:27.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1429863424 bytes 2024-08-31T08:34:27.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1434724352 bytes 2024-08-31T08:34:27.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1524806656 bytes 2024-08-31T08:34:28.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1576624128 bytes 2024-08-31T08:34:28.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1618586624 bytes 2024-08-31T08:34:28.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1637800960 bytes 2024-08-31T08:34:28.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1672226816 bytes 2024-08-31T08:34:28.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1695301632 bytes 2024-08-31T08:34:28.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1725680640 bytes 2024-08-31T08:34:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1818079232 bytes 2024-08-31T08:34:29.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1864173568 bytes 2024-08-31T08:34:29.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1909506048 bytes 2024-08-31T08:34:29.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1954788352 bytes 2024-08-31T08:34:29.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2008312832 bytes 2024-08-31T08:34:29.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2030489600 bytes 2024-08-31T08:34:30.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2062223360 bytes 2024-08-31T08:34:30.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2111086592 bytes 2024-08-31T08:34:30.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2137100288 bytes 2024-08-31T08:34:30.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2146610176 bytes 2024-08-31T08:34:30.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2220945408 bytes 2024-08-31T08:34:30.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2262121472 bytes 2024-08-31T08:34:31.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2291949568 bytes 2024-08-31T08:34:31.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2292578304 bytes 2024-08-31T08:34:31.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2336328704 bytes 2024-08-31T08:34:31.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2374335488 bytes 2024-08-31T08:34:31.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2418185216 bytes 2024-08-31T08:34:31.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2458575872 bytes 2024-08-31T08:34:31.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2503502848 bytes 2024-08-31T08:34:32.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2584916992 bytes 2024-08-31T08:34:32.296 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2595856384 bytes 2024-08-31T08:34:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2658623488 bytes 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_21 2024-08-31T08:34:32.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_21 128M 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:32.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_21 2024-08-31T08:34:32.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:32.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2024-08-31T08:34:33.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:34:33.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_21': 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1362fa0e4518 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1362fa0e4518 2024-08-31T08:34:33.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:32 2024 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:32 2024 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:32 2024 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 83a21b96-117e-4747-bd14-d641899f4663 2024-08-31T08:34:33.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_21 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:34:33.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:34:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:34:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:34:33.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:34:34.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 39858176 bytes 2024-08-31T08:34:34.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 57371648 bytes 2024-08-31T08:34:34.318 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 113329152 bytes 2024-08-31T08:34:34.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 133460992 bytes 2024-08-31T08:34:34.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 134093824 bytes 2024-08-31T08:34:34.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 158087168 bytes 2024-08-31T08:34:34.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 194031616 bytes 2024-08-31T08:34:34.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 245008384 bytes 2024-08-31T08:34:35.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 306103296 bytes 2024-08-31T08:34:35.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 316578816 bytes 2024-08-31T08:34:35.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 356675584 bytes 2024-08-31T08:34:35.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 393065472 bytes 2024-08-31T08:34:35.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 432128000 bytes 2024-08-31T08:34:36.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 455798784 bytes 2024-08-31T08:34:36.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 469563392 bytes 2024-08-31T08:34:36.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 554957824 bytes 2024-08-31T08:34:36.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 596563968 bytes 2024-08-31T08:34:36.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 598928384 bytes 2024-08-31T08:34:36.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 625444864 bytes 2024-08-31T08:34:36.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 664602624 bytes 2024-08-31T08:34:37.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 741139456 bytes 2024-08-31T08:34:37.243 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7311397686930282) > weight (0.25) 2024-08-31T08:34:37.244 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5478531272828939) > weight (0.25) 2024-08-31T08:34:37.244 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5643154454345202) > weight (0.25) 2024-08-31T08:34:37.244 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.2721600403990272) > weight (0.25) 2024-08-31T08:34:37.244 INFO:tasks.rbd_mirror_thrash:waiting for 81 secs before thrashing 2024-08-31T08:34:37.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 810662912 bytes 2024-08-31T08:34:37.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 885938176 bytes 2024-08-31T08:34:37.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 976871424 bytes 2024-08-31T08:34:38.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1029585920 bytes 2024-08-31T08:34:38.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1040733184 bytes 2024-08-31T08:34:38.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1084967936 bytes 2024-08-31T08:34:38.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1113260032 bytes 2024-08-31T08:34:38.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1142039552 bytes 2024-08-31T08:34:38.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1159200768 bytes 2024-08-31T08:34:39.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1226725376 bytes 2024-08-31T08:34:39.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1248712704 bytes 2024-08-31T08:34:39.319 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1268558848 bytes 2024-08-31T08:34:39.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1326574592 bytes 2024-08-31T08:34:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1354391552 bytes 2024-08-31T08:34:39.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1362637824 bytes 2024-08-31T08:34:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1459148800 bytes 2024-08-31T08:34:40.338 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1468773376 bytes 2024-08-31T08:34:40.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1569760256 bytes 2024-08-31T08:34:40.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1614560256 bytes 2024-08-31T08:34:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1631313920 bytes 2024-08-31T08:34:41.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1723785216 bytes 2024-08-31T08:34:41.434 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1825236992 bytes 2024-08-31T08:34:41.652 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1884334080 bytes 2024-08-31T08:34:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1944934400 bytes 2024-08-31T08:34:42.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1987740672 bytes 2024-08-31T08:34:42.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2008767488 bytes 2024-08-31T08:34:42.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2052374528 bytes 2024-08-31T08:34:42.432 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2067676160 bytes 2024-08-31T08:34:42.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2128021504 bytes 2024-08-31T08:34:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2179964928 bytes 2024-08-31T08:34:43.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2281876480 bytes 2024-08-31T08:34:43.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2294104064 bytes 2024-08-31T08:34:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2294729728 bytes 2024-08-31T08:34:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2364578816 bytes 2024-08-31T08:34:43.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2390150144 bytes 2024-08-31T08:34:43.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2424774656 bytes 2024-08-31T08:34:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2430111744 bytes 2024-08-31T08:34:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2449317888 bytes 2024-08-31T08:34:44.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2501870592 bytes 2024-08-31T08:34:44.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2520698880 bytes 2024-08-31T08:34:44.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2532169728 bytes 2024-08-31T08:34:44.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2627979264 bytes 2024-08-31T08:34:44.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2653548544 bytes 2024-08-31T08:34:44.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2667110400 bytes 2024-08-31T08:34:45.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2698616832 bytes 2024-08-31T08:34:45.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2743128064 bytes 2024-08-31T08:34:45.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2788790272 bytes 2024-08-31T08:34:45.518 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2791495680 bytes 2024-08-31T08:34:45.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2848883712 bytes 2024-08-31T08:34:45.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2872881152 bytes 2024-08-31T08:34:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2933116928 bytes 2024-08-31T08:34:46.187 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:46.185+0000 7fb313f7e640 -1 reset not still connected to 0x55976f1e4820 2024-08-31T08:34:46.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 2966638592 bytes 2024-08-31T08:34:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 2967248896 bytes 2024-08-31T08:34:46.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3041892352 bytes 2024-08-31T08:34:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:34:46.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_22 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_22 128M 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:34:46.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:34:46.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_22 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2024-08-31T08:34:46.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:46.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:46.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:34:46.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:34:46.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2024-08-31T08:34:47.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:34:47.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2024-08-31T08:34:47.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_22': 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 136e7e99c1dc 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.136e7e99c1dc 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:46 2024 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:46 2024 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:46 2024 2024-08-31T08:34:47.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:34:47.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:34:47.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 500dccbe-6d7c-4195-84c9-4bdc2b07ec24 2024-08-31T08:34:47.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_22 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:34:47.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:34:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=14 2024-08-31T08:34:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:34:47.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:34:48.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 43086848 bytes 2024-08-31T08:34:48.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 74060800 bytes 2024-08-31T08:34:48.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 110146560 bytes 2024-08-31T08:34:48.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 111981568 bytes 2024-08-31T08:34:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 118076416 bytes 2024-08-31T08:34:48.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 150343680 bytes 2024-08-31T08:34:48.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 197291008 bytes 2024-08-31T08:34:49.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 255527936 bytes 2024-08-31T08:34:49.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 337006592 bytes 2024-08-31T08:34:49.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 361881600 bytes 2024-08-31T08:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 414023680 bytes 2024-08-31T08:34:49.805 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.803+0000 7fb313f7e640 -1 reset not still connected to 0x559768a8ed00 2024-08-31T08:34:49.807 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.805+0000 7fb313f7e640 -1 reset not still connected to 0x55976983a820 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.807+0000 7fb313f7e640 -1 reset not still connected to 0x55976983ab60 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x55976b854b60 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86dad0 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x55976b9e65b0 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x55976db9e410 2024-08-31T08:34:49.812 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x559781985110 2024-08-31T08:34:49.813 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fc8f0 2024-08-31T08:34:49.813 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:34:49.810+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdad0 2024-08-31T08:34:49.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 496960512 bytes 2024-08-31T08:34:50.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 576295936 bytes 2024-08-31T08:34:50.270 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 589016064 bytes 2024-08-31T08:34:50.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 634915840 bytes 2024-08-31T08:34:50.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 645232640 bytes 2024-08-31T08:34:50.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 705531904 bytes 2024-08-31T08:34:50.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 721789952 bytes 2024-08-31T08:34:50.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 737228800 bytes 2024-08-31T08:34:51.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 756976640 bytes 2024-08-31T08:34:51.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 795204608 bytes 2024-08-31T08:34:51.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 864060416 bytes 2024-08-31T08:34:51.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 873344000 bytes 2024-08-31T08:34:51.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 952393728 bytes 2024-08-31T08:34:51.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 987922432 bytes 2024-08-31T08:34:52.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1011476480 bytes 2024-08-31T08:34:52.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1077008384 bytes 2024-08-31T08:34:52.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1143296000 bytes 2024-08-31T08:34:52.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1148847104 bytes 2024-08-31T08:34:52.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1173991424 bytes 2024-08-31T08:34:52.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1207114752 bytes 2024-08-31T08:34:53.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1254934528 bytes 2024-08-31T08:34:53.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1290652672 bytes 2024-08-31T08:34:53.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1329481728 bytes 2024-08-31T08:34:53.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1386306560 bytes 2024-08-31T08:34:53.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1463205888 bytes 2024-08-31T08:34:54.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1517416448 bytes 2024-08-31T08:34:54.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1564642304 bytes 2024-08-31T08:34:54.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1601244160 bytes 2024-08-31T08:34:54.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1629047808 bytes 2024-08-31T08:34:54.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1678561280 bytes 2024-08-31T08:34:54.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1731936256 bytes 2024-08-31T08:34:55.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1752878080 bytes 2024-08-31T08:34:55.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1764928512 bytes 2024-08-31T08:34:55.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1767596032 bytes 2024-08-31T08:34:55.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1801004032 bytes 2024-08-31T08:34:55.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1861011456 bytes 2024-08-31T08:34:55.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1923211264 bytes 2024-08-31T08:34:56.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2023271424 bytes 2024-08-31T08:34:56.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2074093568 bytes 2024-08-31T08:34:56.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2104283136 bytes 2024-08-31T08:34:56.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2148653056 bytes 2024-08-31T08:34:56.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2212288512 bytes 2024-08-31T08:34:57.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2251000832 bytes 2024-08-31T08:34:57.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2265196544 bytes 2024-08-31T08:34:57.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2314931200 bytes 2024-08-31T08:34:57.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2335691776 bytes 2024-08-31T08:34:57.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2393639936 bytes 2024-08-31T08:34:57.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2413372416 bytes 2024-08-31T08:34:58.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2470047744 bytes 2024-08-31T08:34:58.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2539662336 bytes 2024-08-31T08:34:58.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2606990336 bytes 2024-08-31T08:34:58.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2645680128 bytes 2024-08-31T08:34:58.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2692289536 bytes 2024-08-31T08:34:59.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2725096448 bytes 2024-08-31T08:34:59.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2766922752 bytes 2024-08-31T08:34:59.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2798570496 bytes 2024-08-31T08:34:59.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2811175936 bytes 2024-08-31T08:34:59.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2825578496 bytes 2024-08-31T08:34:59.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2893830144 bytes 2024-08-31T08:35:00.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2944211968 bytes 2024-08-31T08:35:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2955809792 bytes 2024-08-31T08:35:00.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3028923392 bytes 2024-08-31T08:35:00.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3084618752 bytes 2024-08-31T08:35:00.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3184041984 bytes 2024-08-31T08:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3212151808 bytes 2024-08-31T08:35:01.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3247708160 bytes 2024-08-31T08:35:01.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3269404672 bytes 2024-08-31T08:35:01.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3332770816 bytes 2024-08-31T08:35:01.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3390345216 bytes 2024-08-31T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:01.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_23 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_23 128M 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_23 2024-08-31T08:35:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:01.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:01.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2024-08-31T08:35:02.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:02.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2024-08-31T08:35:02.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_23': 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 137a8c5967cc 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.137a8c5967cc 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:01 2024 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:01 2024 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:01 2024 2024-08-31T08:35:02.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:02.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:02.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d80f7262-f0bd-4606-9216-6d6e2b02c77a 2024-08-31T08:35:02.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_23 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:02.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 56990720 bytes 2024-08-31T08:35:03.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 117234688 bytes 2024-08-31T08:35:03.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 164934656 bytes 2024-08-31T08:35:03.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 215538688 bytes 2024-08-31T08:35:03.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 242293760 bytes 2024-08-31T08:35:03.780 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 300611584 bytes 2024-08-31T08:35:03.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 369424384 bytes 2024-08-31T08:35:04.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 446658560 bytes 2024-08-31T08:35:04.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 468728832 bytes 2024-08-31T08:35:04.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 488157184 bytes 2024-08-31T08:35:04.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 574597120 bytes 2024-08-31T08:35:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 590520320 bytes 2024-08-31T08:35:05.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 630635520 bytes 2024-08-31T08:35:05.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 673346560 bytes 2024-08-31T08:35:05.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 730150912 bytes 2024-08-31T08:35:05.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 795366400 bytes 2024-08-31T08:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 825494528 bytes 2024-08-31T08:35:05.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 870341632 bytes 2024-08-31T08:35:06.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 949364736 bytes 2024-08-31T08:35:06.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 1032164352 bytes 2024-08-31T08:35:06.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1111197696 bytes 2024-08-31T08:35:06.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1183835136 bytes 2024-08-31T08:35:07.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1270039552 bytes 2024-08-31T08:35:07.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1283240960 bytes 2024-08-31T08:35:07.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1310747648 bytes 2024-08-31T08:35:07.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1376237568 bytes 2024-08-31T08:35:07.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1422318592 bytes 2024-08-31T08:35:08.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1433153536 bytes 2024-08-31T08:35:08.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1476037632 bytes 2024-08-31T08:35:08.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1526591488 bytes 2024-08-31T08:35:08.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1543789568 bytes 2024-08-31T08:35:08.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1558279168 bytes 2024-08-31T08:35:08.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1613937664 bytes 2024-08-31T08:35:08.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1636279296 bytes 2024-08-31T08:35:09.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1666809856 bytes 2024-08-31T08:35:09.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1702873088 bytes 2024-08-31T08:35:09.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1703486464 bytes 2024-08-31T08:35:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1730993152 bytes 2024-08-31T08:35:09.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1855712256 bytes 2024-08-31T08:35:10.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1901436928 bytes 2024-08-31T08:35:10.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 2003214336 bytes 2024-08-31T08:35:10.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 2051841024 bytes 2024-08-31T08:35:10.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2110729216 bytes 2024-08-31T08:35:10.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2114395136 bytes 2024-08-31T08:35:11.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2173957120 bytes 2024-08-31T08:35:11.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2267059200 bytes 2024-08-31T08:35:11.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2299487232 bytes 2024-08-31T08:35:11.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2371820544 bytes 2024-08-31T08:35:11.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2404288512 bytes 2024-08-31T08:35:11.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2419582976 bytes 2024-08-31T08:35:12.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2448229376 bytes 2024-08-31T08:35:12.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2557134848 bytes 2024-08-31T08:35:12.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2614249472 bytes 2024-08-31T08:35:12.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_24 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_24 128M 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:12.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:12.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:12.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_24 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:12.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2024-08-31T08:35:13.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:13.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2024-08-31T08:35:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_24': 2024-08-31T08:35:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:13.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 138622d41ffa 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.138622d41ffa 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:12 2024 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:12 2024 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:12 2024 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: fca44610-a28e-42e0-b629-6dcbe524a929 2024-08-31T08:35:13.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_24 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:35:13.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:13.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:35:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:13.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:13.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 36507648 bytes 2024-08-31T08:35:14.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 83991552 bytes 2024-08-31T08:35:14.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 97247232 bytes 2024-08-31T08:35:14.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 103518208 bytes 2024-08-31T08:35:14.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 136664064 bytes 2024-08-31T08:35:14.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 170739712 bytes 2024-08-31T08:35:14.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 207065088 bytes 2024-08-31T08:35:14.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 282699776 bytes 2024-08-31T08:35:15.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 337400832 bytes 2024-08-31T08:35:15.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 395604992 bytes 2024-08-31T08:35:15.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 422059008 bytes 2024-08-31T08:35:15.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 476531712 bytes 2024-08-31T08:35:15.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 488904704 bytes 2024-08-31T08:35:16.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 593222656 bytes 2024-08-31T08:35:16.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 643943424 bytes 2024-08-31T08:35:16.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 739923968 bytes 2024-08-31T08:35:16.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 783973376 bytes 2024-08-31T08:35:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 853991424 bytes 2024-08-31T08:35:17.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 906065920 bytes 2024-08-31T08:35:17.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 955821056 bytes 2024-08-31T08:35:17.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 965662720 bytes 2024-08-31T08:35:17.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 988838912 bytes 2024-08-31T08:35:17.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1016019968 bytes 2024-08-31T08:35:17.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1051127808 bytes 2024-08-31T08:35:18.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1076962304 bytes 2024-08-31T08:35:18.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1110972416 bytes 2024-08-31T08:35:18.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1122219008 bytes 2024-08-31T08:35:18.339 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7896272865494371) > weight (0.25) 2024-08-31T08:35:18.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5465695754345894) > weight (0.25) 2024-08-31T08:35:18.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7672421387409333) > weight (0.25) 2024-08-31T08:35:18.340 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.3109182203161257) > weight (0.25) 2024-08-31T08:35:18.340 INFO:tasks.rbd_mirror_thrash:waiting for 106 secs before thrashing 2024-08-31T08:35:18.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1181257728 bytes 2024-08-31T08:35:18.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1227127808 bytes 2024-08-31T08:35:18.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1294769152 bytes 2024-08-31T08:35:19.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1343560704 bytes 2024-08-31T08:35:19.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1395081216 bytes 2024-08-31T08:35:19.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1418960896 bytes 2024-08-31T08:35:19.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1436841984 bytes 2024-08-31T08:35:19.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1488390144 bytes 2024-08-31T08:35:19.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1494203392 bytes 2024-08-31T08:35:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1530689536 bytes 2024-08-31T08:35:20.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1598941184 bytes 2024-08-31T08:35:20.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1636640768 bytes 2024-08-31T08:35:20.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1671662592 bytes 2024-08-31T08:35:20.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1711674368 bytes 2024-08-31T08:35:20.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1736534016 bytes 2024-08-31T08:35:21.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1849728000 bytes 2024-08-31T08:35:21.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1894856704 bytes 2024-08-31T08:35:21.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1942620160 bytes 2024-08-31T08:35:21.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2008578048 bytes 2024-08-31T08:35:21.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2038956032 bytes 2024-08-31T08:35:22.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2076378112 bytes 2024-08-31T08:35:22.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2120043520 bytes 2024-08-31T08:35:22.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2167114752 bytes 2024-08-31T08:35:22.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2228327424 bytes 2024-08-31T08:35:22.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2264955904 bytes 2024-08-31T08:35:23.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2275502080 bytes 2024-08-31T08:35:23.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2319014912 bytes 2024-08-31T08:35:23.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2370981888 bytes 2024-08-31T08:35:23.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2409473024 bytes 2024-08-31T08:35:23.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2445481984 bytes 2024-08-31T08:35:23.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2472107008 bytes 2024-08-31T08:35:24.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2498799616 bytes 2024-08-31T08:35:24.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2550178816 bytes 2024-08-31T08:35:24.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2588452864 bytes 2024-08-31T08:35:24.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2612663296 bytes 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_25 2024-08-31T08:35:24.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_25 128M 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:24.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_25 2024-08-31T08:35:24.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:24.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:24.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:24.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:24.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2024-08-31T08:35:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:24.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_25': 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13926bf4a6f 2024-08-31T08:35:24.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13926bf4a6f 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:24 2024 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:24 2024 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:24 2024 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 8ee04712-4bc0-4a3d-a3bf-ac9627495068 2024-08-31T08:35:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_25 2024-08-31T08:35:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:35:24.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:35:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:24.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 15043584 bytes 2024-08-31T08:35:25.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 41553920 bytes 2024-08-31T08:35:25.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 42180608 bytes 2024-08-31T08:35:25.355 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 73000960 bytes 2024-08-31T08:35:25.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 111772672 bytes 2024-08-31T08:35:25.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 177655808 bytes 2024-08-31T08:35:25.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 262854656 bytes 2024-08-31T08:35:26.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 329018368 bytes 2024-08-31T08:35:26.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 365861888 bytes 2024-08-31T08:35:26.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 389198848 bytes 2024-08-31T08:35:26.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 389823488 bytes 2024-08-31T08:35:26.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 420999168 bytes 2024-08-31T08:35:26.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 474014720 bytes 2024-08-31T08:35:27.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 498452480 bytes 2024-08-31T08:35:27.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 547006464 bytes 2024-08-31T08:35:27.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 603971584 bytes 2024-08-31T08:35:27.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 632221696 bytes 2024-08-31T08:35:27.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 691366912 bytes 2024-08-31T08:35:27.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 702707712 bytes 2024-08-31T08:35:28.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 727198720 bytes 2024-08-31T08:35:28.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 803566592 bytes 2024-08-31T08:35:28.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 830836736 bytes 2024-08-31T08:35:28.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 862081024 bytes 2024-08-31T08:35:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 879625216 bytes 2024-08-31T08:35:28.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 917615616 bytes 2024-08-31T08:35:29.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 926854144 bytes 2024-08-31T08:35:29.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 973730816 bytes 2024-08-31T08:35:29.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 991830016 bytes 2024-08-31T08:35:29.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1039755264 bytes 2024-08-31T08:35:29.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1044940800 bytes 2024-08-31T08:35:29.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1103856640 bytes 2024-08-31T08:35:30.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1198878720 bytes 2024-08-31T08:35:30.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1216181248 bytes 2024-08-31T08:35:30.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1280571392 bytes 2024-08-31T08:35:30.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1296371712 bytes 2024-08-31T08:35:30.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1363392512 bytes 2024-08-31T08:35:31.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1451344896 bytes 2024-08-31T08:35:31.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1485181952 bytes 2024-08-31T08:35:31.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1531180032 bytes 2024-08-31T08:35:31.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1556889600 bytes 2024-08-31T08:35:31.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1603202048 bytes 2024-08-31T08:35:31.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1670071296 bytes 2024-08-31T08:35:32.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1744084992 bytes 2024-08-31T08:35:32.270 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1774326784 bytes 2024-08-31T08:35:32.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1860787200 bytes 2024-08-31T08:35:32.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1955611648 bytes 2024-08-31T08:35:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1992587264 bytes 2024-08-31T08:35:33.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2015939584 bytes 2024-08-31T08:35:33.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2064403456 bytes 2024-08-31T08:35:33.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2133324800 bytes 2024-08-31T08:35:33.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2151647232 bytes 2024-08-31T08:35:33.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2179425280 bytes 2024-08-31T08:35:33.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2201734144 bytes 2024-08-31T08:35:34.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2225843200 bytes 2024-08-31T08:35:34.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2253756416 bytes 2024-08-31T08:35:34.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2283496448 bytes 2024-08-31T08:35:34.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2299097088 bytes 2024-08-31T08:35:34.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2322526208 bytes 2024-08-31T08:35:34.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2372161536 bytes 2024-08-31T08:35:34.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2380814336 bytes 2024-08-31T08:35:35.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2445631488 bytes 2024-08-31T08:35:35.254 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2471498752 bytes 2024-08-31T08:35:35.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2538214400 bytes 2024-08-31T08:35:35.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2588364800 bytes 2024-08-31T08:35:35.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2613028864 bytes 2024-08-31T08:35:35.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_26 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:35:35.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_26 128M 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:35.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:35.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:35.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_26 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:35:35.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:35.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2024-08-31T08:35:36.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:36.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_26': 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 139e8debcd16 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.139e8debcd16 2024-08-31T08:35:36.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:35 2024 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:35 2024 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:35 2024 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b5b3e638-f5aa-43cf-a2f1-98602c0046a0 2024-08-31T08:35:36.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_26 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:35:36.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:36.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:35:36.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:36.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 78795776 bytes 2024-08-31T08:35:37.430 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 165302272 bytes 2024-08-31T08:35:37.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 182862848 bytes 2024-08-31T08:35:37.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 222411776 bytes 2024-08-31T08:35:37.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 292028416 bytes 2024-08-31T08:35:38.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 312624128 bytes 2024-08-31T08:35:38.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 313234432 bytes 2024-08-31T08:35:38.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 401842176 bytes 2024-08-31T08:35:38.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 472700928 bytes 2024-08-31T08:35:38.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 532272128 bytes 2024-08-31T08:35:39.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 577815552 bytes 2024-08-31T08:35:39.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 624461824 bytes 2024-08-31T08:35:39.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 671961088 bytes 2024-08-31T08:35:39.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 741813248 bytes 2024-08-31T08:35:39.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 791554048 bytes 2024-08-31T08:35:40.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 835270656 bytes 2024-08-31T08:35:40.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 868252672 bytes 2024-08-31T08:35:40.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 906300416 bytes 2024-08-31T08:35:40.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 912205824 bytes 2024-08-31T08:35:40.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 968499200 bytes 2024-08-31T08:35:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1004404736 bytes 2024-08-31T08:35:41.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1035072512 bytes 2024-08-31T08:35:41.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1081568256 bytes 2024-08-31T08:35:41.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1130725376 bytes 2024-08-31T08:35:41.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1148532736 bytes 2024-08-31T08:35:41.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1214058496 bytes 2024-08-31T08:35:41.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1236905984 bytes 2024-08-31T08:35:42.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1313587200 bytes 2024-08-31T08:35:42.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1405373440 bytes 2024-08-31T08:35:42.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1475604480 bytes 2024-08-31T08:35:42.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1542444032 bytes 2024-08-31T08:35:42.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1563209728 bytes 2024-08-31T08:35:43.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1577559040 bytes 2024-08-31T08:35:43.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1593289728 bytes 2024-08-31T08:35:43.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1654290432 bytes 2024-08-31T08:35:43.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1676014592 bytes 2024-08-31T08:35:43.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1692645376 bytes 2024-08-31T08:35:43.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1729019904 bytes 2024-08-31T08:35:43.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1773990912 bytes 2024-08-31T08:35:44.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1796571136 bytes 2024-08-31T08:35:44.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1870161920 bytes 2024-08-31T08:35:44.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1920158720 bytes 2024-08-31T08:35:44.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1962945536 bytes 2024-08-31T08:35:44.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1994665984 bytes 2024-08-31T08:35:44.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2039347200 bytes 2024-08-31T08:35:45.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2045615104 bytes 2024-08-31T08:35:45.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2110295040 bytes 2024-08-31T08:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2175789056 bytes 2024-08-31T08:35:45.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2194875392 bytes 2024-08-31T08:35:45.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2210799616 bytes 2024-08-31T08:35:45.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2262189056 bytes 2024-08-31T08:35:46.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2262801408 bytes 2024-08-31T08:35:46.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2306971648 bytes 2024-08-31T08:35:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2308088832 bytes 2024-08-31T08:35:46.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2308722688 bytes 2024-08-31T08:35:46.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2383347712 bytes 2024-08-31T08:35:46.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2432941056 bytes 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_27 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_27 128M 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:46.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_27 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:46.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2024-08-31T08:35:47.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:47.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2024-08-31T08:35:47.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_27': 2024-08-31T08:35:47.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:47.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13aacded89dd 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13aacded89dd 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:46 2024 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:46 2024 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:46 2024 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:47.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 23512681-0d1b-4424-93c6-c223988299d9 2024-08-31T08:35:47.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:47.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_27 2024-08-31T08:35:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:35:47.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:47.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:35:47.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:47.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:47.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 22529024 bytes 2024-08-31T08:35:47.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 50060288 bytes 2024-08-31T08:35:48.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 54137856 bytes 2024-08-31T08:35:48.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 103535616 bytes 2024-08-31T08:35:48.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 186099712 bytes 2024-08-31T08:35:48.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 199262208 bytes 2024-08-31T08:35:48.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 218093568 bytes 2024-08-31T08:35:48.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 269266944 bytes 2024-08-31T08:35:49.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 305805312 bytes 2024-08-31T08:35:49.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 319751168 bytes 2024-08-31T08:35:49.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 345041920 bytes 2024-08-31T08:35:49.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 504014848 bytes 2024-08-31T08:35:49.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 564321280 bytes 2024-08-31T08:35:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 573526016 bytes 2024-08-31T08:35:50.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 584483840 bytes 2024-08-31T08:35:50.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 599658496 bytes 2024-08-31T08:35:50.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 630208512 bytes 2024-08-31T08:35:50.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 654709760 bytes 2024-08-31T08:35:50.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 813529088 bytes 2024-08-31T08:35:51.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 867338240 bytes 2024-08-31T08:35:51.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 902208512 bytes 2024-08-31T08:35:51.432 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 930206720 bytes 2024-08-31T08:35:51.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 971049984 bytes 2024-08-31T08:35:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1029995520 bytes 2024-08-31T08:35:52.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1100291072 bytes 2024-08-31T08:35:52.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1116662784 bytes 2024-08-31T08:35:52.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1142841344 bytes 2024-08-31T08:35:52.430 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1157837824 bytes 2024-08-31T08:35:52.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1214437376 bytes 2024-08-31T08:35:52.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1234949120 bytes 2024-08-31T08:35:52.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1252652032 bytes 2024-08-31T08:35:53.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1316371456 bytes 2024-08-31T08:35:53.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1409556480 bytes 2024-08-31T08:35:53.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1431007232 bytes 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:35:53.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_28 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:35:53.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_28 128M 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:35:53.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_28 2024-08-31T08:35:53.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:35:53.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:53.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:35:53.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:35:53.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2024-08-31T08:35:53.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:53.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:53.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:35:53.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:35:53.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2024-08-31T08:35:54.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:35:54.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_28': 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13b6c540830c 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13b6c540830c 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:35:54.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:53 2024 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:53 2024 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:53 2024 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 32c899e0-aef6-4039-9156-ccc4a4e46708 2024-08-31T08:35:54.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_28 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:35:54.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:35:54.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:35:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:35:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:35:54.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 41675776 bytes 2024-08-31T08:35:54.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 64123904 bytes 2024-08-31T08:35:55.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 101139456 bytes 2024-08-31T08:35:55.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 151406592 bytes 2024-08-31T08:35:55.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 178834432 bytes 2024-08-31T08:35:55.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 222451712 bytes 2024-08-31T08:35:55.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 286845952 bytes 2024-08-31T08:35:56.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 327811072 bytes 2024-08-31T08:35:56.246 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 376417280 bytes 2024-08-31T08:35:56.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 398126080 bytes 2024-08-31T08:35:56.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 475628544 bytes 2024-08-31T08:35:56.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 536825856 bytes 2024-08-31T08:35:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 550815744 bytes 2024-08-31T08:35:57.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 653969408 bytes 2024-08-31T08:35:57.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 666686464 bytes 2024-08-31T08:35:57.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 687762432 bytes 2024-08-31T08:35:57.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 773204992 bytes 2024-08-31T08:35:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 804528128 bytes 2024-08-31T08:35:58.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 825420800 bytes 2024-08-31T08:35:58.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5201029578011007) > weight (0.25) 2024-08-31T08:35:58.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7779907787600044) > weight (0.25) 2024-08-31T08:35:58.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.31536167025247086) > weight (0.25) 2024-08-31T08:35:58.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8862498670528344) > weight (0.25) 2024-08-31T08:35:58.245 INFO:tasks.rbd_mirror_thrash:waiting for 64 secs before thrashing 2024-08-31T08:35:58.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 878891008 bytes 2024-08-31T08:35:58.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 879505408 bytes 2024-08-31T08:35:58.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 894014464 bytes 2024-08-31T08:35:58.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 894633984 bytes 2024-08-31T08:35:58.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 960086016 bytes 2024-08-31T08:35:58.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 986633216 bytes 2024-08-31T08:35:59.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1024866304 bytes 2024-08-31T08:35:59.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1027339264 bytes 2024-08-31T08:35:59.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1112621056 bytes 2024-08-31T08:35:59.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1127998464 bytes 2024-08-31T08:35:59.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1167549440 bytes 2024-08-31T08:35:59.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1212974080 bytes 2024-08-31T08:36:00.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1271037952 bytes 2024-08-31T08:36:00.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1365902336 bytes 2024-08-31T08:36:00.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1451820032 bytes 2024-08-31T08:36:00.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1509059584 bytes 2024-08-31T08:36:01.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1526501376 bytes 2024-08-31T08:36:01.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1543187456 bytes 2024-08-31T08:36:01.318 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1564667904 bytes 2024-08-31T08:36:01.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1612730368 bytes 2024-08-31T08:36:01.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1701229568 bytes 2024-08-31T08:36:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1763918848 bytes 2024-08-31T08:36:02.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1803588608 bytes 2024-08-31T08:36:02.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1822231552 bytes 2024-08-31T08:36:02.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1835977728 bytes 2024-08-31T08:36:02.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1867524096 bytes 2024-08-31T08:36:02.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1877987328 bytes 2024-08-31T08:36:03.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1934100480 bytes 2024-08-31T08:36:03.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1935980544 bytes 2024-08-31T08:36:03.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1957565440 bytes 2024-08-31T08:36:03.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2037882880 bytes 2024-08-31T08:36:03.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2050119680 bytes 2024-08-31T08:36:03.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2094472192 bytes 2024-08-31T08:36:03.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2126689280 bytes 2024-08-31T08:36:04.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2173741056 bytes 2024-08-31T08:36:04.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2214110208 bytes 2024-08-31T08:36:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2234074112 bytes 2024-08-31T08:36:04.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2263792640 bytes 2024-08-31T08:36:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:04.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_29 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:04.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_29 128M 2024-08-31T08:36:04.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:04.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_29 2024-08-31T08:36:04.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:04.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2024-08-31T08:36:05.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:05.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2024-08-31T08:36:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_29': 2024-08-31T08:36:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13c29ab63783 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13c29ab63783 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:04 2024 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:04 2024 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:04 2024 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b9fca043-4527-4ec0-8fb0-13a1a765c247 2024-08-31T08:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:05.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_29 2024-08-31T08:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=5 2024-08-31T08:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:05.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 4538368 bytes 2024-08-31T08:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 81223680 bytes 2024-08-31T08:36:06.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 98882560 bytes 2024-08-31T08:36:06.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 138711040 bytes 2024-08-31T08:36:06.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 159472640 bytes 2024-08-31T08:36:06.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 185968640 bytes 2024-08-31T08:36:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 271286272 bytes 2024-08-31T08:36:07.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 360161280 bytes 2024-08-31T08:36:07.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 391739392 bytes 2024-08-31T08:36:07.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 404953088 bytes 2024-08-31T08:36:07.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 454166528 bytes 2024-08-31T08:36:07.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 475191296 bytes 2024-08-31T08:36:07.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 494798848 bytes 2024-08-31T08:36:08.171 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 589836288 bytes 2024-08-31T08:36:08.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 603977728 bytes 2024-08-31T08:36:08.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 704354304 bytes 2024-08-31T08:36:08.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 713694208 bytes 2024-08-31T08:36:08.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 781072384 bytes 2024-08-31T08:36:09.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 834207744 bytes 2024-08-31T08:36:09.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 851845120 bytes 2024-08-31T08:36:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 869938176 bytes 2024-08-31T08:36:09.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 934875136 bytes 2024-08-31T08:36:09.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 980869120 bytes 2024-08-31T08:36:09.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 981491712 bytes 2024-08-31T08:36:10.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1034972160 bytes 2024-08-31T08:36:10.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1064059904 bytes 2024-08-31T08:36:10.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1120208896 bytes 2024-08-31T08:36:10.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1166782464 bytes 2024-08-31T08:36:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:10.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_30 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:10.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_30 128M 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:36:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_30 2024-08-31T08:36:10.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:10.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2024-08-31T08:36:11.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:11.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_30': 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13cee85c9276 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13cee85c9276 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:11.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:10 2024 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:10 2024 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:10 2024 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: f528088b-901a-428c-946e-f8201633ca46 2024-08-31T08:36:11.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_30 2024-08-31T08:36:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:11.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:36:11.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:36:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:11.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:12.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 76427264 bytes 2024-08-31T08:36:12.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 115895296 bytes 2024-08-31T08:36:12.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 166159360 bytes 2024-08-31T08:36:12.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 229021696 bytes 2024-08-31T08:36:12.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 298758144 bytes 2024-08-31T08:36:12.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 319927296 bytes 2024-08-31T08:36:13.142 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 350699520 bytes 2024-08-31T08:36:13.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 417769472 bytes 2024-08-31T08:36:13.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 452466688 bytes 2024-08-31T08:36:13.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 489549824 bytes 2024-08-31T08:36:13.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 570571776 bytes 2024-08-31T08:36:14.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 630808576 bytes 2024-08-31T08:36:14.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 653496320 bytes 2024-08-31T08:36:14.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 660003840 bytes 2024-08-31T08:36:14.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 675930112 bytes 2024-08-31T08:36:14.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 686039040 bytes 2024-08-31T08:36:14.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 751938560 bytes 2024-08-31T08:36:15.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 899673088 bytes 2024-08-31T08:36:15.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 951755776 bytes 2024-08-31T08:36:15.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 996002816 bytes 2024-08-31T08:36:15.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1006309376 bytes 2024-08-31T08:36:16.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1066516480 bytes 2024-08-31T08:36:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1101196288 bytes 2024-08-31T08:36:16.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1125485568 bytes 2024-08-31T08:36:16.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1144459264 bytes 2024-08-31T08:36:16.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1182538752 bytes 2024-08-31T08:36:16.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1214808064 bytes 2024-08-31T08:36:16.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1221834752 bytes 2024-08-31T08:36:17.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1266566144 bytes 2024-08-31T08:36:17.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1304441856 bytes 2024-08-31T08:36:17.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1363859456 bytes 2024-08-31T08:36:17.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1387353088 bytes 2024-08-31T08:36:17.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1407461376 bytes 2024-08-31T08:36:17.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1436929024 bytes 2024-08-31T08:36:17.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1457573888 bytes 2024-08-31T08:36:18.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1472452608 bytes 2024-08-31T08:36:18.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1504306176 bytes 2024-08-31T08:36:18.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1520793600 bytes 2024-08-31T08:36:18.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1581232128 bytes 2024-08-31T08:36:18.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1633463296 bytes 2024-08-31T08:36:19.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1698327552 bytes 2024-08-31T08:36:19.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1727800320 bytes 2024-08-31T08:36:19.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1793736704 bytes 2024-08-31T08:36:19.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1889974272 bytes 2024-08-31T08:36:19.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1906643968 bytes 2024-08-31T08:36:19.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1948234752 bytes 2024-08-31T08:36:20.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1959982080 bytes 2024-08-31T08:36:20.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2025815040 bytes 2024-08-31T08:36:20.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2066982912 bytes 2024-08-31T08:36:20.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2105837568 bytes 2024-08-31T08:36:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:20.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_31 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_31 128M 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:20.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_31 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2024-08-31T08:36:20.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:20.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:20.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:36:20.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:20.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2024-08-31T08:36:21.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:21.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2024-08-31T08:36:21.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_31': 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13da51c6f12a 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13da51c6f12a 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:20 2024 2024-08-31T08:36:21.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:20 2024 2024-08-31T08:36:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:20 2024 2024-08-31T08:36:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:21.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 681647ab-f5ff-4fcf-8d6a-32f3fe7fa4ba 2024-08-31T08:36:21.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_31 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:21.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:36:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:21.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:21.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 10939392 bytes 2024-08-31T08:36:22.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 80069632 bytes 2024-08-31T08:36:22.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 125835264 bytes 2024-08-31T08:36:22.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 207384576 bytes 2024-08-31T08:36:22.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 229136384 bytes 2024-08-31T08:36:22.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 285273088 bytes 2024-08-31T08:36:23.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 344569856 bytes 2024-08-31T08:36:23.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 378639360 bytes 2024-08-31T08:36:23.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 397893632 bytes 2024-08-31T08:36:23.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 414882816 bytes 2024-08-31T08:36:23.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 435134464 bytes 2024-08-31T08:36:23.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 474697728 bytes 2024-08-31T08:36:24.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 537908224 bytes 2024-08-31T08:36:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 587493376 bytes 2024-08-31T08:36:24.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 588103680 bytes 2024-08-31T08:36:24.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 597683200 bytes 2024-08-31T08:36:24.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 635009024 bytes 2024-08-31T08:36:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 700633088 bytes 2024-08-31T08:36:25.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 807603200 bytes 2024-08-31T08:36:25.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 867454976 bytes 2024-08-31T08:36:25.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 908168192 bytes 2024-08-31T08:36:25.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 936266752 bytes 2024-08-31T08:36:25.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 986233856 bytes 2024-08-31T08:36:25.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1002072064 bytes 2024-08-31T08:36:26.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1060598784 bytes 2024-08-31T08:36:26.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1061218304 bytes 2024-08-31T08:36:26.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1066592256 bytes 2024-08-31T08:36:26.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1084492800 bytes 2024-08-31T08:36:26.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1132030976 bytes 2024-08-31T08:36:26.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1176619008 bytes 2024-08-31T08:36:27.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1239070720 bytes 2024-08-31T08:36:27.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1306945536 bytes 2024-08-31T08:36:27.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1324057600 bytes 2024-08-31T08:36:27.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1360612352 bytes 2024-08-31T08:36:27.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1361229824 bytes 2024-08-31T08:36:28.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1444211712 bytes 2024-08-31T08:36:28.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1551686656 bytes 2024-08-31T08:36:28.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1588074496 bytes 2024-08-31T08:36:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:28.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_32 2024-08-31T08:36:28.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:36:28.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_32 128M 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:28.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:28.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:28.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:28.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:28.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_32 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:28.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:36:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:28.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2024-08-31T08:36:29.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:29.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_32': 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13e69c5e03db 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13e69c5e03db 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:29.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:28 2024 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:28 2024 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:28 2024 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: f34b2723-009a-42b7-ad38-540c04928627 2024-08-31T08:36:29.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_32 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:36:29.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:29.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=8 2024-08-31T08:36:29.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:29.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:30.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 58472448 bytes 2024-08-31T08:36:30.278 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 82075648 bytes 2024-08-31T08:36:30.433 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 123109376 bytes 2024-08-31T08:36:30.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 209685504 bytes 2024-08-31T08:36:30.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 288695296 bytes 2024-08-31T08:36:31.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 308878336 bytes 2024-08-31T08:36:31.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 342291456 bytes 2024-08-31T08:36:31.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 369484800 bytes 2024-08-31T08:36:31.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 371048448 bytes 2024-08-31T08:36:31.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 412167168 bytes 2024-08-31T08:36:31.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 414176256 bytes 2024-08-31T08:36:31.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 447389696 bytes 2024-08-31T08:36:32.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 495005696 bytes 2024-08-31T08:36:32.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 556365824 bytes 2024-08-31T08:36:32.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 610375680 bytes 2024-08-31T08:36:32.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 644797440 bytes 2024-08-31T08:36:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 698124288 bytes 2024-08-31T08:36:33.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 736967680 bytes 2024-08-31T08:36:33.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 767477760 bytes 2024-08-31T08:36:33.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 807606272 bytes 2024-08-31T08:36:33.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 861747200 bytes 2024-08-31T08:36:33.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 958322688 bytes 2024-08-31T08:36:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1001311232 bytes 2024-08-31T08:36:34.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1083675648 bytes 2024-08-31T08:36:34.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1151166464 bytes 2024-08-31T08:36:34.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1205353472 bytes 2024-08-31T08:36:34.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1228662784 bytes 2024-08-31T08:36:35.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1310543872 bytes 2024-08-31T08:36:35.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1367133184 bytes 2024-08-31T08:36:35.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1403971584 bytes 2024-08-31T08:36:35.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1469074432 bytes 2024-08-31T08:36:35.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1487198208 bytes 2024-08-31T08:36:36.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1574469632 bytes 2024-08-31T08:36:36.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1644481536 bytes 2024-08-31T08:36:36.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1694031872 bytes 2024-08-31T08:36:36.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1694659584 bytes 2024-08-31T08:36:36.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1720890368 bytes 2024-08-31T08:36:37.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1747885056 bytes 2024-08-31T08:36:37.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1820357632 bytes 2024-08-31T08:36:37.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1899010048 bytes 2024-08-31T08:36:37.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1946356736 bytes 2024-08-31T08:36:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:37.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_33 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:37.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_33 128M 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:37.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_33 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2024-08-31T08:36:37.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:36:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:37.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2024-08-31T08:36:38.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:38.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_33': 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13f29aad920 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13f29aad920 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:38.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:37 2024 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:37 2024 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:37 2024 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 88c939cc-d595-4585-959a-55085fcd0473 2024-08-31T08:36:38.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_33 2024-08-31T08:36:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:38.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:38.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:36:38.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:36:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:38.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:39.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 38910976 bytes 2024-08-31T08:36:39.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 67786752 bytes 2024-08-31T08:36:39.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 113284096 bytes 2024-08-31T08:36:39.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 125504512 bytes 2024-08-31T08:36:39.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 188153856 bytes 2024-08-31T08:36:39.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 240158720 bytes 2024-08-31T08:36:40.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 262958080 bytes 2024-08-31T08:36:40.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 316455936 bytes 2024-08-31T08:36:40.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 348491776 bytes 2024-08-31T08:36:40.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 405297152 bytes 2024-08-31T08:36:40.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 439871488 bytes 2024-08-31T08:36:41.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 488099840 bytes 2024-08-31T08:36:41.266 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 555718656 bytes 2024-08-31T08:36:41.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 599656448 bytes 2024-08-31T08:36:41.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 600481792 bytes 2024-08-31T08:36:41.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 660687872 bytes 2024-08-31T08:36:42.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 765840384 bytes 2024-08-31T08:36:42.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 802595840 bytes 2024-08-31T08:36:42.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 826230784 bytes 2024-08-31T08:36:42.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 862630912 bytes 2024-08-31T08:36:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 904203264 bytes 2024-08-31T08:36:42.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 929420288 bytes 2024-08-31T08:36:43.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1021357056 bytes 2024-08-31T08:36:43.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1099079680 bytes 2024-08-31T08:36:43.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1124043776 bytes 2024-08-31T08:36:43.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1135545344 bytes 2024-08-31T08:36:43.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1186062336 bytes 2024-08-31T08:36:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1211148288 bytes 2024-08-31T08:36:44.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1260902400 bytes 2024-08-31T08:36:44.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1308875776 bytes 2024-08-31T08:36:44.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1314670592 bytes 2024-08-31T08:36:44.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1406401536 bytes 2024-08-31T08:36:44.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1419336704 bytes 2024-08-31T08:36:45.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1482152960 bytes 2024-08-31T08:36:45.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1563761664 bytes 2024-08-31T08:36:45.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1601585152 bytes 2024-08-31T08:36:45.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1645929472 bytes 2024-08-31T08:36:45.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1674289152 bytes 2024-08-31T08:36:45.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1738903552 bytes 2024-08-31T08:36:46.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1763609600 bytes 2024-08-31T08:36:46.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1777748992 bytes 2024-08-31T08:36:46.379 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1825598464 bytes 2024-08-31T08:36:46.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1841845248 bytes 2024-08-31T08:36:46.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1883678720 bytes 2024-08-31T08:36:46.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1956994048 bytes 2024-08-31T08:36:47.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1990053888 bytes 2024-08-31T08:36:47.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2039097344 bytes 2024-08-31T08:36:47.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2096615424 bytes 2024-08-31T08:36:47.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2107257856 bytes 2024-08-31T08:36:47.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2120283136 bytes 2024-08-31T08:36:47.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2148014080 bytes 2024-08-31T08:36:47.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2162040832 bytes 2024-08-31T08:36:47.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2162654208 bytes 2024-08-31T08:36:48.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2184156160 bytes 2024-08-31T08:36:48.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2189969408 bytes 2024-08-31T08:36:48.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2238224384 bytes 2024-08-31T08:36:48.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2295145472 bytes 2024-08-31T08:36:48.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2355610624 bytes 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_34 2024-08-31T08:36:48.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_34 128M 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:36:48.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_34 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:48.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:36:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:36:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2024-08-31T08:36:49.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:36:49.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_34': 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 13fe1e38d176 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.13fe1e38d176 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:36:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:48 2024 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:48 2024 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:48 2024 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d946a5bb-2842-4293-bff6-1ff598883e15 2024-08-31T08:36:49.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_34 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:36:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:36:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:36:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:36:49.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:36:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 24556544 bytes 2024-08-31T08:36:49.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 53553152 bytes 2024-08-31T08:36:50.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 133193728 bytes 2024-08-31T08:36:50.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 157891584 bytes 2024-08-31T08:36:50.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 186251264 bytes 2024-08-31T08:36:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 246093824 bytes 2024-08-31T08:36:50.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 334006272 bytes 2024-08-31T08:36:51.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 378767360 bytes 2024-08-31T08:36:51.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 394318848 bytes 2024-08-31T08:36:51.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 428968960 bytes 2024-08-31T08:36:51.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 436465664 bytes 2024-08-31T08:36:51.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 580410368 bytes 2024-08-31T08:36:52.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 656436224 bytes 2024-08-31T08:36:52.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 673528832 bytes 2024-08-31T08:36:52.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 711363584 bytes 2024-08-31T08:36:52.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 739240960 bytes 2024-08-31T08:36:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 814970880 bytes 2024-08-31T08:36:53.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 887221248 bytes 2024-08-31T08:36:53.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 888244224 bytes 2024-08-31T08:36:53.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 978273280 bytes 2024-08-31T08:36:53.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1026736128 bytes 2024-08-31T08:36:53.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1085520896 bytes 2024-08-31T08:36:53.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1110055936 bytes 2024-08-31T08:36:54.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1174414336 bytes 2024-08-31T08:36:54.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1186583552 bytes 2024-08-31T08:36:54.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1248749568 bytes 2024-08-31T08:36:54.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1370140672 bytes 2024-08-31T08:36:55.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1406625792 bytes 2024-08-31T08:36:55.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1493521408 bytes 2024-08-31T08:36:55.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1535533056 bytes 2024-08-31T08:36:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1536139264 bytes 2024-08-31T08:36:55.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1565759488 bytes 2024-08-31T08:36:55.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1599177728 bytes 2024-08-31T08:36:56.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1599801344 bytes 2024-08-31T08:36:56.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1639665664 bytes 2024-08-31T08:36:56.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1670516736 bytes 2024-08-31T08:36:56.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1739408384 bytes 2024-08-31T08:36:56.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1778973696 bytes 2024-08-31T08:36:56.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1791662080 bytes 2024-08-31T08:36:57.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1827093504 bytes 2024-08-31T08:36:57.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1944547328 bytes 2024-08-31T08:36:57.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1988310016 bytes 2024-08-31T08:36:57.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2025146368 bytes 2024-08-31T08:36:57.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2050010112 bytes 2024-08-31T08:36:58.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2113128448 bytes 2024-08-31T08:36:58.347 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2155925504 bytes 2024-08-31T08:36:58.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2174527488 bytes 2024-08-31T08:36:58.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2201241600 bytes 2024-08-31T08:36:58.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2201875456 bytes 2024-08-31T08:36:58.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2247020544 bytes 2024-08-31T08:36:59.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2329745408 bytes 2024-08-31T08:36:59.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2348278784 bytes 2024-08-31T08:36:59.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2358962176 bytes 2024-08-31T08:36:59.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2361407488 bytes 2024-08-31T08:36:59.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2460991488 bytes 2024-08-31T08:36:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2486193152 bytes 2024-08-31T08:37:00.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2506403840 bytes 2024-08-31T08:37:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2571266048 bytes 2024-08-31T08:37:00.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2592611328 bytes 2024-08-31T08:37:00.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2633443328 bytes 2024-08-31T08:37:00.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2657088512 bytes 2024-08-31T08:37:00.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2691258368 bytes 2024-08-31T08:37:01.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2748612608 bytes 2024-08-31T08:37:01.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2774315008 bytes 2024-08-31T08:37:01.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2819246080 bytes 2024-08-31T08:37:01.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2861829120 bytes 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:01.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_35 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_35 128M 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_35 2024-08-31T08:37:01.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:01.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2024-08-31T08:37:02.247 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-08-31T08:37:02.248 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-08-31T08:37:02.248 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:37:02.249 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:37:02.248+0000 7f651cd0d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 30970) UID: 1000 2024-08-31T08:37:02.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:02.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_35': 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 140a75ba250e 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.140a75ba250e 2024-08-31T08:37:02.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:01 2024 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:01 2024 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:01 2024 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c96b3a90-a801-484f-ba1a-ada1712c4cb4 2024-08-31T08:37:02.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_35 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:37:02.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:37:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:02.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:02.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 49288192 bytes 2024-08-31T08:37:03.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 124399616 bytes 2024-08-31T08:37:03.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 201095168 bytes 2024-08-31T08:37:03.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 201699328 bytes 2024-08-31T08:37:03.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 252602368 bytes 2024-08-31T08:37:03.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 261585920 bytes 2024-08-31T08:37:03.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 263352320 bytes 2024-08-31T08:37:04.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 326080512 bytes 2024-08-31T08:37:04.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 340330496 bytes 2024-08-31T08:37:04.341 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7930783515595639) > weight (0.25) 2024-08-31T08:37:04.341 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9269176739159125) > weight (0.25) 2024-08-31T08:37:04.341 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.606547878480488) > weight (0.25) 2024-08-31T08:37:04.341 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7750070653673583) > weight (0.25) 2024-08-31T08:37:04.341 INFO:tasks.rbd_mirror_thrash:waiting for 92 secs before thrashing 2024-08-31T08:37:04.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 391346176 bytes 2024-08-31T08:37:04.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 451953664 bytes 2024-08-31T08:37:04.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 474284032 bytes 2024-08-31T08:37:05.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 535743488 bytes 2024-08-31T08:37:05.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 574467072 bytes 2024-08-31T08:37:05.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 643880960 bytes 2024-08-31T08:37:05.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 724038656 bytes 2024-08-31T08:37:05.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 725660672 bytes 2024-08-31T08:37:06.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 791429120 bytes 2024-08-31T08:37:06.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 805424128 bytes 2024-08-31T08:37:06.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 835179520 bytes 2024-08-31T08:37:06.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 836077568 bytes 2024-08-31T08:37:06.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 880295936 bytes 2024-08-31T08:37:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 981711872 bytes 2024-08-31T08:37:07.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1033090048 bytes 2024-08-31T08:37:07.248 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-08-31T08:37:07.249 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:37:07.249 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-08-31T08:37:07.249 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-08-31T08:37:07.249 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-08-31T08:37:07.254 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-08-31T08:37:07.254 INFO:tasks.rbd_mirror_thrash:waiting for 96 secs before thrashing 2024-08-31T08:37:07.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1054357504 bytes 2024-08-31T08:37:07.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1090465792 bytes 2024-08-31T08:37:07.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1158392832 bytes 2024-08-31T08:37:07.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1179871232 bytes 2024-08-31T08:37:08.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1237878784 bytes 2024-08-31T08:37:08.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1277421568 bytes 2024-08-31T08:37:08.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1331994624 bytes 2024-08-31T08:37:08.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1412310016 bytes 2024-08-31T08:37:08.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1464128512 bytes 2024-08-31T08:37:09.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1475936256 bytes 2024-08-31T08:37:09.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1540744192 bytes 2024-08-31T08:37:09.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1584227328 bytes 2024-08-31T08:37:09.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1592583168 bytes 2024-08-31T08:37:09.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1640029184 bytes 2024-08-31T08:37:09.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1655931904 bytes 2024-08-31T08:37:10.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1718302720 bytes 2024-08-31T08:37:10.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1743187968 bytes 2024-08-31T08:37:10.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1768123392 bytes 2024-08-31T08:37:10.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1785970688 bytes 2024-08-31T08:37:10.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1834597376 bytes 2024-08-31T08:37:11.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1909313536 bytes 2024-08-31T08:37:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1980700672 bytes 2024-08-31T08:37:11.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2016775168 bytes 2024-08-31T08:37:11.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2091326464 bytes 2024-08-31T08:37:11.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_36 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_36 128M 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:37:11.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:11.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_36 2024-08-31T08:37:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:11.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2024-08-31T08:37:12.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:12.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_36': 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 141bcf37906d 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.141bcf37906d 2024-08-31T08:37:12.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:11 2024 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:11 2024 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:11 2024 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 7c94795d-bc10-44e4-9ddb-b555399db9b4 2024-08-31T08:37:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:12.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_36 2024-08-31T08:37:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:37:12.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:37:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:12.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:12.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 11990016 bytes 2024-08-31T08:37:13.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 86879232 bytes 2024-08-31T08:37:13.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 97376256 bytes 2024-08-31T08:37:13.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 149593088 bytes 2024-08-31T08:37:13.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 151395328 bytes 2024-08-31T08:37:13.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 170578944 bytes 2024-08-31T08:37:13.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 220820480 bytes 2024-08-31T08:37:14.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 275341312 bytes 2024-08-31T08:37:14.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 287326208 bytes 2024-08-31T08:37:14.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 290610176 bytes 2024-08-31T08:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 386585600 bytes 2024-08-31T08:37:14.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 389891072 bytes 2024-08-31T08:37:14.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 461212672 bytes 2024-08-31T08:37:15.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 501542912 bytes 2024-08-31T08:37:15.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 550154240 bytes 2024-08-31T08:37:15.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 612062208 bytes 2024-08-31T08:37:15.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 620468224 bytes 2024-08-31T08:37:15.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 676353024 bytes 2024-08-31T08:37:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 721540096 bytes 2024-08-31T08:37:16.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 734301184 bytes 2024-08-31T08:37:16.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 757616640 bytes 2024-08-31T08:37:16.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 758798336 bytes 2024-08-31T08:37:16.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 771823616 bytes 2024-08-31T08:37:16.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 826400768 bytes 2024-08-31T08:37:16.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 900321280 bytes 2024-08-31T08:37:16.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 908440576 bytes 2024-08-31T08:37:17.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 922842112 bytes 2024-08-31T08:37:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 939236352 bytes 2024-08-31T08:37:17.319 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 988356608 bytes 2024-08-31T08:37:17.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1054494720 bytes 2024-08-31T08:37:17.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1061018624 bytes 2024-08-31T08:37:17.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1121652736 bytes 2024-08-31T08:37:18.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1147212800 bytes 2024-08-31T08:37:18.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1190046720 bytes 2024-08-31T08:37:18.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1266273280 bytes 2024-08-31T08:37:18.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1304484864 bytes 2024-08-31T08:37:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_37 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_37 128M 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:18.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_37 2024-08-31T08:37:18.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:18.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2024-08-31T08:37:19.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:19.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_37': 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 14233f9c6217 2024-08-31T08:37:19.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.14233f9c6217 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:18 2024 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:18 2024 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:18 2024 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: eb84190b-e551-45d5-8d58-e362047a6f7d 2024-08-31T08:37:19.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_37 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:37:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:19.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:37:19.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:19.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:20.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 13318144 bytes 2024-08-31T08:37:20.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 31804416 bytes 2024-08-31T08:37:20.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 38554624 bytes 2024-08-31T08:37:20.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 94329856 bytes 2024-08-31T08:37:20.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 145045504 bytes 2024-08-31T08:37:20.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 187520000 bytes 2024-08-31T08:37:21.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 276661248 bytes 2024-08-31T08:37:21.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 319541248 bytes 2024-08-31T08:37:21.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 362842112 bytes 2024-08-31T08:37:21.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 408864768 bytes 2024-08-31T08:37:21.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 464576512 bytes 2024-08-31T08:37:21.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 483592192 bytes 2024-08-31T08:37:22.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 560961536 bytes 2024-08-31T08:37:22.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 586530816 bytes 2024-08-31T08:37:22.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 596409344 bytes 2024-08-31T08:37:22.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 631023616 bytes 2024-08-31T08:37:22.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 693583872 bytes 2024-08-31T08:37:22.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 732105728 bytes 2024-08-31T08:37:23.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 789494784 bytes 2024-08-31T08:37:23.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 809556992 bytes 2024-08-31T08:37:23.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 895799296 bytes 2024-08-31T08:37:23.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 951834624 bytes 2024-08-31T08:37:23.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1006516224 bytes 2024-08-31T08:37:24.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1032708096 bytes 2024-08-31T08:37:24.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1080790016 bytes 2024-08-31T08:37:24.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1158609920 bytes 2024-08-31T08:37:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1215613952 bytes 2024-08-31T08:37:24.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1250638848 bytes 2024-08-31T08:37:25.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1354418176 bytes 2024-08-31T08:37:25.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1417947136 bytes 2024-08-31T08:37:25.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1487174656 bytes 2024-08-31T08:37:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1494786048 bytes 2024-08-31T08:37:26.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1579939840 bytes 2024-08-31T08:37:26.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1597947904 bytes 2024-08-31T08:37:26.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1618529280 bytes 2024-08-31T08:37:26.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1683959808 bytes 2024-08-31T08:37:26.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1684561920 bytes 2024-08-31T08:37:26.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1700221952 bytes 2024-08-31T08:37:26.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1738100736 bytes 2024-08-31T08:37:27.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1795948544 bytes 2024-08-31T08:37:27.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1847387136 bytes 2024-08-31T08:37:27.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1867423744 bytes 2024-08-31T08:37:27.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1896320000 bytes 2024-08-31T08:37:27.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1964327936 bytes 2024-08-31T08:37:28.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2044188672 bytes 2024-08-31T08:37:28.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2061416448 bytes 2024-08-31T08:37:28.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2107684864 bytes 2024-08-31T08:37:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2131112960 bytes 2024-08-31T08:37:28.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2161201152 bytes 2024-08-31T08:37:28.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2183365632 bytes 2024-08-31T08:37:28.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2199747584 bytes 2024-08-31T08:37:29.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2226644992 bytes 2024-08-31T08:37:29.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2263085056 bytes 2024-08-31T08:37:29.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2301480960 bytes 2024-08-31T08:37:29.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2393147392 bytes 2024-08-31T08:37:29.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2445073408 bytes 2024-08-31T08:37:30.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2504760320 bytes 2024-08-31T08:37:30.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2581065728 bytes 2024-08-31T08:37:30.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2625934336 bytes 2024-08-31T08:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2643928064 bytes 2024-08-31T08:37:30.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2667530240 bytes 2024-08-31T08:37:30.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2680785920 bytes 2024-08-31T08:37:31.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2712902656 bytes 2024-08-31T08:37:31.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2791194624 bytes 2024-08-31T08:37:31.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2798938112 bytes 2024-08-31T08:37:31.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2856872960 bytes 2024-08-31T08:37:31.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2884076544 bytes 2024-08-31T08:37:31.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2900912128 bytes 2024-08-31T08:37:31.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2912500736 bytes 2024-08-31T08:37:32.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2941778944 bytes 2024-08-31T08:37:32.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2982785024 bytes 2024-08-31T08:37:32.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3003266048 bytes 2024-08-31T08:37:32.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3016367104 bytes 2024-08-31T08:37:32.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3083514880 bytes 2024-08-31T08:37:32.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:32.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_38 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_38 128M 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:32.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_38 2024-08-31T08:37:32.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:32.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:32.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:32.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:32.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2024-08-31T08:37:33.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:33.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2024-08-31T08:37:33.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_38': 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 142ce3e2e15a 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.142ce3e2e15a 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:32 2024 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:32 2024 2024-08-31T08:37:33.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:32 2024 2024-08-31T08:37:33.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:33.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:33.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94f6c5db-e9bd-4250-8d91-441f4a234f60 2024-08-31T08:37:33.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_38 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:37:33.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:37:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:33.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:34.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 30671872 bytes 2024-08-31T08:37:34.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 31304704 bytes 2024-08-31T08:37:34.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 84350976 bytes 2024-08-31T08:37:34.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 146162688 bytes 2024-08-31T08:37:34.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 180924416 bytes 2024-08-31T08:37:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 199856128 bytes 2024-08-31T08:37:35.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 225576960 bytes 2024-08-31T08:37:35.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 288826368 bytes 2024-08-31T08:37:35.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 314261504 bytes 2024-08-31T08:37:35.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 340291584 bytes 2024-08-31T08:37:35.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 407877632 bytes 2024-08-31T08:37:35.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 432812032 bytes 2024-08-31T08:37:36.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 454087680 bytes 2024-08-31T08:37:36.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 460877824 bytes 2024-08-31T08:37:36.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 503386112 bytes 2024-08-31T08:37:36.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 543156224 bytes 2024-08-31T08:37:36.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 610061312 bytes 2024-08-31T08:37:36.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 642188288 bytes 2024-08-31T08:37:37.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 659886080 bytes 2024-08-31T08:37:37.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 694265856 bytes 2024-08-31T08:37:37.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 714186752 bytes 2024-08-31T08:37:37.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 781373440 bytes 2024-08-31T08:37:37.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 814807040 bytes 2024-08-31T08:37:37.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 879024128 bytes 2024-08-31T08:37:38.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 901972992 bytes 2024-08-31T08:37:38.316 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 983401472 bytes 2024-08-31T08:37:38.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1029866496 bytes 2024-08-31T08:37:38.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1098474496 bytes 2024-08-31T08:37:38.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1099544576 bytes 2024-08-31T08:37:39.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1119452160 bytes 2024-08-31T08:37:39.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1136245760 bytes 2024-08-31T08:37:39.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1205095424 bytes 2024-08-31T08:37:39.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1229555712 bytes 2024-08-31T08:37:39.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1281308672 bytes 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.791+0000 7fb313f7e640 -1 reset not still connected to 0x559768a8fad0 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976983a820 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976983ab60 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976b854b60 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86cd00 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86dad0 2024-08-31T08:37:39.794 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.792+0000 7fb313f7e640 -1 reset not still connected to 0x55976b9e65b0 2024-08-31T08:37:39.800 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.798+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d1860 2024-08-31T08:37:39.800 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.798+0000 7fb313f7e640 -1 reset not still connected to 0x559781985110 2024-08-31T08:37:39.800 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.798+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fc8f0 2024-08-31T08:37:39.800 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.798+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdad0 2024-08-31T08:37:39.800 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.798+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdd40 2024-08-31T08:37:39.802 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.799+0000 7fb313f7e640 -1 reset not still connected to 0x559795b389c0 2024-08-31T08:37:39.802 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.799+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fc340 2024-08-31T08:37:39.802 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:39.799+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fdee0 2024-08-31T08:37:39.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1306031104 bytes 2024-08-31T08:37:40.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1361007616 bytes 2024-08-31T08:37:40.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1375787008 bytes 2024-08-31T08:37:40.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1451715584 bytes 2024-08-31T08:37:40.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1482951680 bytes 2024-08-31T08:37:40.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1554977792 bytes 2024-08-31T08:37:41.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1618193408 bytes 2024-08-31T08:37:41.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1643319296 bytes 2024-08-31T08:37:41.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1667794944 bytes 2024-08-31T08:37:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1723676672 bytes 2024-08-31T08:37:41.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1772210176 bytes 2024-08-31T08:37:41.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1845728256 bytes 2024-08-31T08:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1863988224 bytes 2024-08-31T08:37:42.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1875543040 bytes 2024-08-31T08:37:42.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1901116416 bytes 2024-08-31T08:37:42.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1922463744 bytes 2024-08-31T08:37:42.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1938885632 bytes 2024-08-31T08:37:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2002482176 bytes 2024-08-31T08:37:42.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_39 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_39 128M 2024-08-31T08:37:42.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:42.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_39 2024-08-31T08:37:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:42.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:42.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2024-08-31T08:37:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:43.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_39': 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1435d3c27d5d 2024-08-31T08:37:43.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1435d3c27d5d 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:42 2024 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:42 2024 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:42 2024 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6cf73648-3155-40fd-aa07-805332949efa 2024-08-31T08:37:43.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:43.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_39 2024-08-31T08:37:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:37:43.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:37:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:43.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 27057152 bytes 2024-08-31T08:37:43.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 50070528 bytes 2024-08-31T08:37:44.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 68222976 bytes 2024-08-31T08:37:44.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 91417600 bytes 2024-08-31T08:37:44.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 92051456 bytes 2024-08-31T08:37:44.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 92644352 bytes 2024-08-31T08:37:44.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 144805888 bytes 2024-08-31T08:37:44.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 145424384 bytes 2024-08-31T08:37:44.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 169348096 bytes 2024-08-31T08:37:44.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 257932288 bytes 2024-08-31T08:37:45.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 313064448 bytes 2024-08-31T08:37:45.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 328615936 bytes 2024-08-31T08:37:45.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 374569984 bytes 2024-08-31T08:37:45.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 398720000 bytes 2024-08-31T08:37:45.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 416674816 bytes 2024-08-31T08:37:45.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 470628352 bytes 2024-08-31T08:37:46.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 529230848 bytes 2024-08-31T08:37:46.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 567595008 bytes 2024-08-31T08:37:46.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 576775168 bytes 2024-08-31T08:37:46.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 615922688 bytes 2024-08-31T08:37:46.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 680654848 bytes 2024-08-31T08:37:46.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 754335744 bytes 2024-08-31T08:37:47.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 848186368 bytes 2024-08-31T08:37:47.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 884893696 bytes 2024-08-31T08:37:47.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 956658688 bytes 2024-08-31T08:37:47.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 957265920 bytes 2024-08-31T08:37:47.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1009837056 bytes 2024-08-31T08:37:48.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1010445312 bytes 2024-08-31T08:37:48.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1028098048 bytes 2024-08-31T08:37:48.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1040394240 bytes 2024-08-31T08:37:48.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1070729216 bytes 2024-08-31T08:37:48.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1102173184 bytes 2024-08-31T08:37:48.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1132894208 bytes 2024-08-31T08:37:48.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1141622784 bytes 2024-08-31T08:37:48.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1163971584 bytes 2024-08-31T08:37:49.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1212291072 bytes 2024-08-31T08:37:49.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1243602944 bytes 2024-08-31T08:37:49.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1244220416 bytes 2024-08-31T08:37:49.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1305538560 bytes 2024-08-31T08:37:49.814 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976983a820 2024-08-31T08:37:49.814 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976983aa90 2024-08-31T08:37:49.814 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976983ab60 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976b854b60 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86c9c0 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86cd00 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86dad0 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976b9e65b0 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55976db9e410 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d1860 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x559781985110 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fc8f0 2024-08-31T08:37:49.815 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdad0 2024-08-31T08:37:49.816 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdd40 2024-08-31T08:37:49.816 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x559795b389c0 2024-08-31T08:37:49.816 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fc340 2024-08-31T08:37:49.816 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:37:49.812+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fdee0 2024-08-31T08:37:49.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1374575616 bytes 2024-08-31T08:37:50.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1404923904 bytes 2024-08-31T08:37:50.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1427236864 bytes 2024-08-31T08:37:50.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1485112320 bytes 2024-08-31T08:37:50.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1485731840 bytes 2024-08-31T08:37:50.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1587034112 bytes 2024-08-31T08:37:50.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1613316096 bytes 2024-08-31T08:37:51.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1715162112 bytes 2024-08-31T08:37:51.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1780968448 bytes 2024-08-31T08:37:51.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1819147264 bytes 2024-08-31T08:37:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1890143232 bytes 2024-08-31T08:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1956615168 bytes 2024-08-31T08:37:52.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2006835200 bytes 2024-08-31T08:37:52.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2064364544 bytes 2024-08-31T08:37:52.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2087970816 bytes 2024-08-31T08:37:52.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2126307328 bytes 2024-08-31T08:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2189442048 bytes 2024-08-31T08:37:53.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2254990336 bytes 2024-08-31T08:37:53.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2298376192 bytes 2024-08-31T08:37:53.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2304440320 bytes 2024-08-31T08:37:53.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2342816768 bytes 2024-08-31T08:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2363301888 bytes 2024-08-31T08:37:54.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2462051328 bytes 2024-08-31T08:37:54.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2517384192 bytes 2024-08-31T08:37:54.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2555542528 bytes 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_40 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_40 128M 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:37:54.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:37:54.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:37:54.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:37:54.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:37:54.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_40 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:37:54.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:37:54.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2024-08-31T08:37:55.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:37:55.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_40': 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 143e1b48939b 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.143e1b48939b 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:37:55.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:54 2024 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:54 2024 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:54 2024 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: a0ff3644-b10e-4591-be88-e027ee2d4543 2024-08-31T08:37:55.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_40 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:37:55.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:37:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:37:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:37:55.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:37:55.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 99991552 bytes 2024-08-31T08:37:56.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 125064192 bytes 2024-08-31T08:37:56.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 142932992 bytes 2024-08-31T08:37:56.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 153569280 bytes 2024-08-31T08:37:56.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 195121152 bytes 2024-08-31T08:37:56.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 236949504 bytes 2024-08-31T08:37:56.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 329074688 bytes 2024-08-31T08:37:57.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 384283648 bytes 2024-08-31T08:37:57.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 419294208 bytes 2024-08-31T08:37:57.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 453528576 bytes 2024-08-31T08:37:57.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 489142272 bytes 2024-08-31T08:37:57.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 525312000 bytes 2024-08-31T08:37:58.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 577371136 bytes 2024-08-31T08:37:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 600210432 bytes 2024-08-31T08:37:58.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 642871296 bytes 2024-08-31T08:37:58.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 682372096 bytes 2024-08-31T08:37:58.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 705547264 bytes 2024-08-31T08:37:58.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 775837696 bytes 2024-08-31T08:37:59.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 809789440 bytes 2024-08-31T08:37:59.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 844773376 bytes 2024-08-31T08:37:59.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 852868096 bytes 2024-08-31T08:37:59.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 914789376 bytes 2024-08-31T08:37:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 939014144 bytes 2024-08-31T08:37:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1021853696 bytes 2024-08-31T08:38:00.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1059348480 bytes 2024-08-31T08:38:00.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1078361088 bytes 2024-08-31T08:38:00.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1097819136 bytes 2024-08-31T08:38:00.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1137939456 bytes 2024-08-31T08:38:00.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1153400832 bytes 2024-08-31T08:38:00.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1239595008 bytes 2024-08-31T08:38:01.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1240196096 bytes 2024-08-31T08:38:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1285846016 bytes 2024-08-31T08:38:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1294637056 bytes 2024-08-31T08:38:01.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1316349952 bytes 2024-08-31T08:38:01.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1363600384 bytes 2024-08-31T08:38:01.746 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1383887872 bytes 2024-08-31T08:38:01.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1397341184 bytes 2024-08-31T08:38:02.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1467592704 bytes 2024-08-31T08:38:02.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1492292608 bytes 2024-08-31T08:38:02.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1519734784 bytes 2024-08-31T08:38:02.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1537793024 bytes 2024-08-31T08:38:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1577887744 bytes 2024-08-31T08:38:03.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1609972736 bytes 2024-08-31T08:38:03.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1666556928 bytes 2024-08-31T08:38:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1710777344 bytes 2024-08-31T08:38:03.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1746323456 bytes 2024-08-31T08:38:03.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1797182464 bytes 2024-08-31T08:38:03.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1862019072 bytes 2024-08-31T08:38:04.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1909619712 bytes 2024-08-31T08:38:04.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1932067840 bytes 2024-08-31T08:38:04.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1944020992 bytes 2024-08-31T08:38:04.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 1968728064 bytes 2024-08-31T08:38:04.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2054172672 bytes 2024-08-31T08:38:05.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2107732992 bytes 2024-08-31T08:38:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2159700992 bytes 2024-08-31T08:38:05.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2197315584 bytes 2024-08-31T08:38:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2314638336 bytes 2024-08-31T08:38:06.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2363188224 bytes 2024-08-31T08:38:06.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2417698816 bytes 2024-08-31T08:38:06.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2464246784 bytes 2024-08-31T08:38:06.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2507229184 bytes 2024-08-31T08:38:06.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2540623872 bytes 2024-08-31T08:38:06.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2599399424 bytes 2024-08-31T08:38:07.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2647932928 bytes 2024-08-31T08:38:07.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2648541184 bytes 2024-08-31T08:38:07.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2714110976 bytes 2024-08-31T08:38:07.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_41 2024-08-31T08:38:07.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:38:07.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_41 128M 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:07.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:07.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:07.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_41 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:07.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:38:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:07.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2024-08-31T08:38:08.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:08.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2024-08-31T08:38:08.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_41': 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 144687ddc1d0 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.144687ddc1d0 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:07 2024 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:07 2024 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:07 2024 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 8a05271f-7b34-4139-a110-04860d7d6a27 2024-08-31T08:38:08.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_41 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:38:08.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:08.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:38:08.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:08.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:08.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 34092032 bytes 2024-08-31T08:38:09.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 39521280 bytes 2024-08-31T08:38:09.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 85212160 bytes 2024-08-31T08:38:09.380 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 99284992 bytes 2024-08-31T08:38:09.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 102670336 bytes 2024-08-31T08:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 183822336 bytes 2024-08-31T08:38:10.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 281117696 bytes 2024-08-31T08:38:10.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 317478912 bytes 2024-08-31T08:38:10.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 361584640 bytes 2024-08-31T08:38:10.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 434770944 bytes 2024-08-31T08:38:10.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 480889856 bytes 2024-08-31T08:38:10.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 491633664 bytes 2024-08-31T08:38:11.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 514629632 bytes 2024-08-31T08:38:11.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 521010176 bytes 2024-08-31T08:38:11.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 549356544 bytes 2024-08-31T08:38:11.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 560532480 bytes 2024-08-31T08:38:11.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 590613504 bytes 2024-08-31T08:38:11.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 678345728 bytes 2024-08-31T08:38:11.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 709394432 bytes 2024-08-31T08:38:12.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 798408704 bytes 2024-08-31T08:38:12.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 826173440 bytes 2024-08-31T08:38:12.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 897224704 bytes 2024-08-31T08:38:12.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 982216704 bytes 2024-08-31T08:38:13.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1065312256 bytes 2024-08-31T08:38:13.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1084235776 bytes 2024-08-31T08:38:13.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1139776512 bytes 2024-08-31T08:38:13.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1187820544 bytes 2024-08-31T08:38:13.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1233855488 bytes 2024-08-31T08:38:14.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1271984128 bytes 2024-08-31T08:38:14.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1325765632 bytes 2024-08-31T08:38:14.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1429171200 bytes 2024-08-31T08:38:14.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1449893888 bytes 2024-08-31T08:38:14.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1487345664 bytes 2024-08-31T08:38:14.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1491121152 bytes 2024-08-31T08:38:15.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1541581824 bytes 2024-08-31T08:38:15.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1559372800 bytes 2024-08-31T08:38:15.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1632232448 bytes 2024-08-31T08:38:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1733743616 bytes 2024-08-31T08:38:15.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1738764288 bytes 2024-08-31T08:38:16.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1852863488 bytes 2024-08-31T08:38:16.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1904565248 bytes 2024-08-31T08:38:16.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1973834752 bytes 2024-08-31T08:38:16.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2009330688 bytes 2024-08-31T08:38:17.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2048734208 bytes 2024-08-31T08:38:17.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2049324032 bytes 2024-08-31T08:38:17.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2112352256 bytes 2024-08-31T08:38:17.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2168734720 bytes 2024-08-31T08:38:17.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2207089664 bytes 2024-08-31T08:38:17.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2218548224 bytes 2024-08-31T08:38:18.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2237302784 bytes 2024-08-31T08:38:18.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2297057280 bytes 2024-08-31T08:38:18.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2323441664 bytes 2024-08-31T08:38:18.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2344242176 bytes 2024-08-31T08:38:18.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2374332416 bytes 2024-08-31T08:38:19.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2450356224 bytes 2024-08-31T08:38:19.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2472085504 bytes 2024-08-31T08:38:19.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2494796800 bytes 2024-08-31T08:38:19.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2551189504 bytes 2024-08-31T08:38:19.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2570416128 bytes 2024-08-31T08:38:19.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2661051392 bytes 2024-08-31T08:38:20.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2689797120 bytes 2024-08-31T08:38:20.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2710052864 bytes 2024-08-31T08:38:20.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2739811328 bytes 2024-08-31T08:38:20.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2798282752 bytes 2024-08-31T08:38:20.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2881955840 bytes 2024-08-31T08:38:21.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2889796608 bytes 2024-08-31T08:38:21.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2915018752 bytes 2024-08-31T08:38:21.279 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2934297600 bytes 2024-08-31T08:38:21.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 3021411328 bytes 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_42 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:21.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_42 128M 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:21.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_42 2024-08-31T08:38:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:21.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2024-08-31T08:38:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:38:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:21.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2024-08-31T08:38:22.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:22.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_42': 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 144fc8258bf5 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.144fc8258bf5 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:22.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:21 2024 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:21 2024 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:21 2024 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 65a6d1ca-25ea-415c-b2bd-c2eaed4c4c4d 2024-08-31T08:38:22.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_42 2024-08-31T08:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:22.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:38:22.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:38:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:22.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:23.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 43984896 bytes 2024-08-31T08:38:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 64771072 bytes 2024-08-31T08:38:23.434 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 109050880 bytes 2024-08-31T08:38:23.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 143692800 bytes 2024-08-31T08:38:23.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 216300544 bytes 2024-08-31T08:38:23.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 260538368 bytes 2024-08-31T08:38:24.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 285019136 bytes 2024-08-31T08:38:24.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 333338624 bytes 2024-08-31T08:38:24.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 378784768 bytes 2024-08-31T08:38:24.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 501202944 bytes 2024-08-31T08:38:24.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 521324544 bytes 2024-08-31T08:38:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 542673920 bytes 2024-08-31T08:38:25.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 564748288 bytes 2024-08-31T08:38:25.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 603834368 bytes 2024-08-31T08:38:25.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 700390400 bytes 2024-08-31T08:38:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 796938240 bytes 2024-08-31T08:38:26.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 810321920 bytes 2024-08-31T08:38:26.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 882385920 bytes 2024-08-31T08:38:26.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 915358720 bytes 2024-08-31T08:38:26.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 966512640 bytes 2024-08-31T08:38:27.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1009508352 bytes 2024-08-31T08:38:27.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1064745984 bytes 2024-08-31T08:38:27.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1167514624 bytes 2024-08-31T08:38:27.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1223420928 bytes 2024-08-31T08:38:27.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1259571200 bytes 2024-08-31T08:38:28.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1309009920 bytes 2024-08-31T08:38:28.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1336238080 bytes 2024-08-31T08:38:28.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1393125376 bytes 2024-08-31T08:38:28.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1464436736 bytes 2024-08-31T08:38:28.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1513966592 bytes 2024-08-31T08:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1576528896 bytes 2024-08-31T08:38:29.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1661741056 bytes 2024-08-31T08:38:29.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1709294592 bytes 2024-08-31T08:38:29.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1713293312 bytes 2024-08-31T08:38:29.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1716931584 bytes 2024-08-31T08:38:29.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1789865984 bytes 2024-08-31T08:38:30.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1867485184 bytes 2024-08-31T08:38:30.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1934570496 bytes 2024-08-31T08:38:30.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1960670208 bytes 2024-08-31T08:38:30.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 2016315392 bytes 2024-08-31T08:38:30.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 2038501376 bytes 2024-08-31T08:38:31.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 2127899648 bytes 2024-08-31T08:38:31.250 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2145351680 bytes 2024-08-31T08:38:31.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2162576384 bytes 2024-08-31T08:38:31.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2220711936 bytes 2024-08-31T08:38:31.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2233709568 bytes 2024-08-31T08:38:31.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2267826176 bytes 2024-08-31T08:38:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_43 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_43 128M 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:31.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_43 2024-08-31T08:38:31.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:31.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2024-08-31T08:38:32.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:32.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_43': 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1459d2839c3f 2024-08-31T08:38:32.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1459d2839c3f 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:31 2024 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:31 2024 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:31 2024 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1afe39a8-09d2-4199-bfe9-9908e5a224e5 2024-08-31T08:38:32.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_43 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:38:32.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:32.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:38:32.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:32.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:32.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 44047360 bytes 2024-08-31T08:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 97070080 bytes 2024-08-31T08:38:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 111841280 bytes 2024-08-31T08:38:33.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 147497984 bytes 2024-08-31T08:38:33.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 219338752 bytes 2024-08-31T08:38:33.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 256801792 bytes 2024-08-31T08:38:33.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 286905344 bytes 2024-08-31T08:38:34.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 338616320 bytes 2024-08-31T08:38:34.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 347359232 bytes 2024-08-31T08:38:34.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 394298368 bytes 2024-08-31T08:38:34.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 476470272 bytes 2024-08-31T08:38:34.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 503353344 bytes 2024-08-31T08:38:34.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 532124672 bytes 2024-08-31T08:38:35.142 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 593959936 bytes 2024-08-31T08:38:35.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 650459136 bytes 2024-08-31T08:38:35.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 685113344 bytes 2024-08-31T08:38:35.574 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:38:35.572+0000 7f6cb0353640 -1 reset not still connected to 0x55995436b6c0 2024-08-31T08:38:35.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 687477760 bytes 2024-08-31T08:38:35.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 725756928 bytes 2024-08-31T08:38:35.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 746655744 bytes 2024-08-31T08:38:36.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 770547712 bytes 2024-08-31T08:38:36.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 801347584 bytes 2024-08-31T08:38:36.342 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:38:36.343 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:38:36.343 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-08-31T08:38:36.343 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:38:36.342+0000 7f153b8c8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 32532) UID: 1000 2024-08-31T08:38:36.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 883767296 bytes 2024-08-31T08:38:36.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 920083456 bytes 2024-08-31T08:38:36.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 976215040 bytes 2024-08-31T08:38:37.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 987422720 bytes 2024-08-31T08:38:37.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1028850688 bytes 2024-08-31T08:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1057807360 bytes 2024-08-31T08:38:37.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1130238976 bytes 2024-08-31T08:38:37.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1150872576 bytes 2024-08-31T08:38:37.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1178310656 bytes 2024-08-31T08:38:37.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1179208704 bytes 2024-08-31T08:38:38.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1298034688 bytes 2024-08-31T08:38:38.408 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1334009856 bytes 2024-08-31T08:38:38.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1356490752 bytes 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_44 2024-08-31T08:38:38.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_44 128M 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_44 2024-08-31T08:38:38.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2024-08-31T08:38:39.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:39.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_44': 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 14615fede7c3 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.14615fede7c3 2024-08-31T08:38:39.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:38 2024 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:38 2024 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:38 2024 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1d392638-d8a1-4ad5-a6fb-98b143b11843 2024-08-31T08:38:39.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:39.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_44 2024-08-31T08:38:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:38:39.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=7 2024-08-31T08:38:39.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:39.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:39.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 32019456 bytes 2024-08-31T08:38:40.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 45561856 bytes 2024-08-31T08:38:40.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 116150272 bytes 2024-08-31T08:38:40.344 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:38:40.344 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:38:40.345 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-08-31T08:38:40.345 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:38:40.345 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:38:40.348 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:38:40.348 INFO:tasks.rbd_mirror_thrash:waiting for 89 secs before thrashing 2024-08-31T08:38:40.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 151272448 bytes 2024-08-31T08:38:40.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 186114048 bytes 2024-08-31T08:38:40.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 229135360 bytes 2024-08-31T08:38:40.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 247396352 bytes 2024-08-31T08:38:41.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 277338112 bytes 2024-08-31T08:38:41.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 331594752 bytes 2024-08-31T08:38:41.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 372434944 bytes 2024-08-31T08:38:41.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 389105664 bytes 2024-08-31T08:38:41.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 431168512 bytes 2024-08-31T08:38:42.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 512516096 bytes 2024-08-31T08:38:42.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 554772480 bytes 2024-08-31T08:38:42.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 576727040 bytes 2024-08-31T08:38:42.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 605776896 bytes 2024-08-31T08:38:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 627089408 bytes 2024-08-31T08:38:42.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 661108736 bytes 2024-08-31T08:38:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 675380224 bytes 2024-08-31T08:38:43.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 721207296 bytes 2024-08-31T08:38:43.256 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.37629949233038573) > weight (0.25) 2024-08-31T08:38:43.256 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8708321056165679) > weight (0.25) 2024-08-31T08:38:43.256 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8522574250282985) > weight (0.25) 2024-08-31T08:38:43.256 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.2632749452537957) > weight (0.25) 2024-08-31T08:38:43.256 INFO:tasks.rbd_mirror_thrash:waiting for 68 secs before thrashing 2024-08-31T08:38:43.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 793461760 bytes 2024-08-31T08:38:43.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 879818752 bytes 2024-08-31T08:38:43.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 930802688 bytes 2024-08-31T08:38:44.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 965945344 bytes 2024-08-31T08:38:44.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1053117440 bytes 2024-08-31T08:38:44.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1076476928 bytes 2024-08-31T08:38:44.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1108137984 bytes 2024-08-31T08:38:44.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1144999936 bytes 2024-08-31T08:38:44.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1163695104 bytes 2024-08-31T08:38:45.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1191137280 bytes 2024-08-31T08:38:45.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1264130048 bytes 2024-08-31T08:38:45.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1301370880 bytes 2024-08-31T08:38:45.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1390246912 bytes 2024-08-31T08:38:45.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1390828544 bytes 2024-08-31T08:38:45.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1398659072 bytes 2024-08-31T08:38:46.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1435529216 bytes 2024-08-31T08:38:46.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1467115520 bytes 2024-08-31T08:38:46.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1495987200 bytes 2024-08-31T08:38:46.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1500078080 bytes 2024-08-31T08:38:46.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1547691008 bytes 2024-08-31T08:38:46.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_45 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:46.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_45 128M 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:46.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_45 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:46.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:38:46.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:46.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2024-08-31T08:38:47.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:47.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2024-08-31T08:38:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_45': 2024-08-31T08:38:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 146f23883ae7 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.146f23883ae7 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:46 2024 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:46 2024 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:46 2024 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:47.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 728a938d-32c1-4507-a5bd-b86a667a611c 2024-08-31T08:38:47.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:47.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_45 2024-08-31T08:38:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:38:47.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:47.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=8 2024-08-31T08:38:47.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:47.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 29701120 bytes 2024-08-31T08:38:48.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 54057984 bytes 2024-08-31T08:38:48.319 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 139068416 bytes 2024-08-31T08:38:48.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 193442816 bytes 2024-08-31T08:38:48.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 246120448 bytes 2024-08-31T08:38:48.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 291329024 bytes 2024-08-31T08:38:49.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 363025408 bytes 2024-08-31T08:38:49.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 394731520 bytes 2024-08-31T08:38:49.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 432774144 bytes 2024-08-31T08:38:49.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 447380480 bytes 2024-08-31T08:38:49.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 508093440 bytes 2024-08-31T08:38:49.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 528912384 bytes 2024-08-31T08:38:50.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 532088832 bytes 2024-08-31T08:38:50.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 578629632 bytes 2024-08-31T08:38:50.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 647268352 bytes 2024-08-31T08:38:50.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 701109248 bytes 2024-08-31T08:38:50.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 776894464 bytes 2024-08-31T08:38:51.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 854165504 bytes 2024-08-31T08:38:51.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 899005440 bytes 2024-08-31T08:38:51.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 938860544 bytes 2024-08-31T08:38:51.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1005379584 bytes 2024-08-31T08:38:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1034240000 bytes 2024-08-31T08:38:52.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1076376576 bytes 2024-08-31T08:38:52.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1117852672 bytes 2024-08-31T08:38:52.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1191112704 bytes 2024-08-31T08:38:52.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1272450048 bytes 2024-08-31T08:38:52.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1282160640 bytes 2024-08-31T08:38:53.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1316980736 bytes 2024-08-31T08:38:53.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1382610944 bytes 2024-08-31T08:38:53.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1437632512 bytes 2024-08-31T08:38:53.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1469855744 bytes 2024-08-31T08:38:53.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1501731840 bytes 2024-08-31T08:38:53.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1531268096 bytes 2024-08-31T08:38:54.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1553920000 bytes 2024-08-31T08:38:54.279 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1615651840 bytes 2024-08-31T08:38:54.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1616250880 bytes 2024-08-31T08:38:54.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1631399936 bytes 2024-08-31T08:38:54.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1668658176 bytes 2024-08-31T08:38:54.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1697255424 bytes 2024-08-31T08:38:55.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1755617280 bytes 2024-08-31T08:38:55.278 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1837532160 bytes 2024-08-31T08:38:55.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1936483328 bytes 2024-08-31T08:38:55.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1953138688 bytes 2024-08-31T08:38:55.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:38:55.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_46 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_46 128M 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:38:55.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:38:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:38:55.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_46 2024-08-31T08:38:55.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:38:55.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:38:55.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2024-08-31T08:38:56.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:38:56.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2024-08-31T08:38:56.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_46': 2024-08-31T08:38:56.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:38:56.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:38:56.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 147774f0867b 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.147774f0867b 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:55 2024 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:55 2024 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:55 2024 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: dec3aa5e-1157-4af4-8203-e9297b75a332 2024-08-31T08:38:56.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_46 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:38:56.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:38:56.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:38:56.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 48112640 bytes 2024-08-31T08:38:57.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 85044224 bytes 2024-08-31T08:38:57.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 103006208 bytes 2024-08-31T08:38:57.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 209874944 bytes 2024-08-31T08:38:57.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 216353792 bytes 2024-08-31T08:38:57.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 244721664 bytes 2024-08-31T08:38:57.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 248458240 bytes 2024-08-31T08:38:58.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 254277632 bytes 2024-08-31T08:38:58.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 353171456 bytes 2024-08-31T08:38:58.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 391305216 bytes 2024-08-31T08:38:58.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 463974400 bytes 2024-08-31T08:38:58.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 503567360 bytes 2024-08-31T08:38:59.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 546842624 bytes 2024-08-31T08:38:59.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 610021376 bytes 2024-08-31T08:38:59.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 662811648 bytes 2024-08-31T08:38:59.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 775983104 bytes 2024-08-31T08:38:59.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 812590080 bytes 2024-08-31T08:39:00.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 813207552 bytes 2024-08-31T08:39:00.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 826096640 bytes 2024-08-31T08:39:00.304 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 832759808 bytes 2024-08-31T08:39:00.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 984867840 bytes 2024-08-31T08:39:00.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1018857472 bytes 2024-08-31T08:39:01.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1062690816 bytes 2024-08-31T08:39:01.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1113137152 bytes 2024-08-31T08:39:01.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1120488448 bytes 2024-08-31T08:39:01.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1130086400 bytes 2024-08-31T08:39:01.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1171367936 bytes 2024-08-31T08:39:01.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1171977216 bytes 2024-08-31T08:39:01.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1195466752 bytes 2024-08-31T08:39:01.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1202533376 bytes 2024-08-31T08:39:02.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1295553536 bytes 2024-08-31T08:39:02.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1314149376 bytes 2024-08-31T08:39:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1327014912 bytes 2024-08-31T08:39:02.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1335467008 bytes 2024-08-31T08:39:02.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1440541696 bytes 2024-08-31T08:39:03.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1501969408 bytes 2024-08-31T08:39:03.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1502583808 bytes 2024-08-31T08:39:03.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1545298944 bytes 2024-08-31T08:39:03.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1602425856 bytes 2024-08-31T08:39:03.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1603059712 bytes 2024-08-31T08:39:03.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1672783872 bytes 2024-08-31T08:39:04.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1689163776 bytes 2024-08-31T08:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1698966528 bytes 2024-08-31T08:39:04.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1737976832 bytes 2024-08-31T08:39:04.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1745378304 bytes 2024-08-31T08:39:04.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1770731520 bytes 2024-08-31T08:39:04.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1821965312 bytes 2024-08-31T08:39:04.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1841867776 bytes 2024-08-31T08:39:05.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1904921600 bytes 2024-08-31T08:39:05.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1980547072 bytes 2024-08-31T08:39:05.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2069165056 bytes 2024-08-31T08:39:05.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2113647616 bytes 2024-08-31T08:39:06.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2175324160 bytes 2024-08-31T08:39:06.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2221485056 bytes 2024-08-31T08:39:06.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2254759936 bytes 2024-08-31T08:39:06.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2329223168 bytes 2024-08-31T08:39:06.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2391265280 bytes 2024-08-31T08:39:06.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2391880704 bytes 2024-08-31T08:39:07.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2444608512 bytes 2024-08-31T08:39:07.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2449113088 bytes 2024-08-31T08:39:07.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2499992576 bytes 2024-08-31T08:39:07.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2584986624 bytes 2024-08-31T08:39:07.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2603103232 bytes 2024-08-31T08:39:07.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2613811200 bytes 2024-08-31T08:39:08.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2688312320 bytes 2024-08-31T08:39:08.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2724038656 bytes 2024-08-31T08:39:08.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2724684800 bytes 2024-08-31T08:39:08.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2769733632 bytes 2024-08-31T08:39:08.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2771351552 bytes 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_47 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_47 128M 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:39:08.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:39:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:39:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:39:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:08.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_47 2024-08-31T08:39:08.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:39:08.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:08.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2024-08-31T08:39:09.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:39:09.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2024-08-31T08:39:09.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_47': 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 14813ee673be 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.14813ee673be 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:08 2024 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:08 2024 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:08 2024 2024-08-31T08:39:09.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:39:09.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:39:09.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 47d42bc5-751d-4d5d-bba7-bc3264fb264b 2024-08-31T08:39:09.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_47 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:39:09.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:39:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:39:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:39:09.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:39:10.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 13686784 bytes 2024-08-31T08:39:10.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 73849856 bytes 2024-08-31T08:39:10.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 99192832 bytes 2024-08-31T08:39:10.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 114700288 bytes 2024-08-31T08:39:10.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 127975424 bytes 2024-08-31T08:39:10.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 164889600 bytes 2024-08-31T08:39:10.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 213630976 bytes 2024-08-31T08:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 307616768 bytes 2024-08-31T08:39:11.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 414598144 bytes 2024-08-31T08:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 497466368 bytes 2024-08-31T08:39:11.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 521180160 bytes 2024-08-31T08:39:12.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 631531520 bytes 2024-08-31T08:39:12.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 687674368 bytes 2024-08-31T08:39:12.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 705045504 bytes 2024-08-31T08:39:12.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 764859392 bytes 2024-08-31T08:39:13.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 822351872 bytes 2024-08-31T08:39:13.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 832679936 bytes 2024-08-31T08:39:13.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 885687296 bytes 2024-08-31T08:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 957314048 bytes 2024-08-31T08:39:13.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 994477056 bytes 2024-08-31T08:39:13.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 1006815232 bytes 2024-08-31T08:39:14.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1060597760 bytes 2024-08-31T08:39:14.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1061239808 bytes 2024-08-31T08:39:14.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1113694208 bytes 2024-08-31T08:39:14.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1149752320 bytes 2024-08-31T08:39:14.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1167101952 bytes 2024-08-31T08:39:14.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1231787008 bytes 2024-08-31T08:39:15.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1284159488 bytes 2024-08-31T08:39:15.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1338831872 bytes 2024-08-31T08:39:15.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1403766784 bytes 2024-08-31T08:39:15.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1457956864 bytes 2024-08-31T08:39:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1482483712 bytes 2024-08-31T08:39:16.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1535456256 bytes 2024-08-31T08:39:16.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1620442112 bytes 2024-08-31T08:39:16.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1638326272 bytes 2024-08-31T08:39:16.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1696465920 bytes 2024-08-31T08:39:16.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1719302144 bytes 2024-08-31T08:39:17.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1781145600 bytes 2024-08-31T08:39:17.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1821460480 bytes 2024-08-31T08:39:17.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1866300416 bytes 2024-08-31T08:39:17.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1887607808 bytes 2024-08-31T08:39:17.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1899799552 bytes 2024-08-31T08:39:17.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1963367424 bytes 2024-08-31T08:39:18.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2008222720 bytes 2024-08-31T08:39:18.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2092835840 bytes 2024-08-31T08:39:18.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2130509824 bytes 2024-08-31T08:39:18.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2184677376 bytes 2024-08-31T08:39:18.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2210541568 bytes 2024-08-31T08:39:18.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2233710592 bytes 2024-08-31T08:39:19.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2264133632 bytes 2024-08-31T08:39:19.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2273250304 bytes 2024-08-31T08:39:19.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2367184896 bytes 2024-08-31T08:39:19.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2404478976 bytes 2024-08-31T08:39:19.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2471150592 bytes 2024-08-31T08:39:20.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2556819456 bytes 2024-08-31T08:39:20.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2570382336 bytes 2024-08-31T08:39:20.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2606979072 bytes 2024-08-31T08:39:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2631278592 bytes 2024-08-31T08:39:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2634733568 bytes 2024-08-31T08:39:21.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2740196352 bytes 2024-08-31T08:39:21.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2836792320 bytes 2024-08-31T08:39:21.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2882365440 bytes 2024-08-31T08:39:21.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2920413184 bytes 2024-08-31T08:39:21.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2933614592 bytes 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:39:21.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_48 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_48 128M 2024-08-31T08:39:21.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:21.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_48 2024-08-31T08:39:21.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:21.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2024-08-31T08:39:22.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:39:22.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_48': 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 148b8cdf950a 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.148b8cdf950a 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:39:22.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:21 2024 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:21 2024 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:21 2024 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6639998a-e2c5-467b-9e8b-80fb5fe10b2b 2024-08-31T08:39:22.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:39:22.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:39:22.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_48 2024-08-31T08:39:22.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:22.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:22.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:39:22.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:39:22.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=9 2024-08-31T08:39:22.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:39:22.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:39:22.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 71758848 bytes 2024-08-31T08:39:22.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 72365056 bytes 2024-08-31T08:39:23.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 120864768 bytes 2024-08-31T08:39:23.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 136837120 bytes 2024-08-31T08:39:23.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 137443328 bytes 2024-08-31T08:39:23.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 204263424 bytes 2024-08-31T08:39:23.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 246052864 bytes 2024-08-31T08:39:23.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 282429440 bytes 2024-08-31T08:39:24.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 303694848 bytes 2024-08-31T08:39:24.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 342367232 bytes 2024-08-31T08:39:24.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 372352000 bytes 2024-08-31T08:39:24.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 454627328 bytes 2024-08-31T08:39:24.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 529617920 bytes 2024-08-31T08:39:24.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 535186432 bytes 2024-08-31T08:39:25.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 633511936 bytes 2024-08-31T08:39:25.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 680461312 bytes 2024-08-31T08:39:25.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 721020928 bytes 2024-08-31T08:39:25.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 734328832 bytes 2024-08-31T08:39:25.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 760523776 bytes 2024-08-31T08:39:25.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 787025920 bytes 2024-08-31T08:39:26.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 844155904 bytes 2024-08-31T08:39:26.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 844756992 bytes 2024-08-31T08:39:26.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 876525568 bytes 2024-08-31T08:39:26.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 916600832 bytes 2024-08-31T08:39:26.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 952013824 bytes 2024-08-31T08:39:26.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 993108992 bytes 2024-08-31T08:39:27.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1026746368 bytes 2024-08-31T08:39:27.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1049191424 bytes 2024-08-31T08:39:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1055166464 bytes 2024-08-31T08:39:27.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1103526912 bytes 2024-08-31T08:39:27.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1212681216 bytes 2024-08-31T08:39:28.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1266560000 bytes 2024-08-31T08:39:28.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1316329472 bytes 2024-08-31T08:39:28.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1335502848 bytes 2024-08-31T08:39:28.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1379326976 bytes 2024-08-31T08:39:28.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1442978816 bytes 2024-08-31T08:39:29.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1515871232 bytes 2024-08-31T08:39:29.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1553667072 bytes 2024-08-31T08:39:29.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1605551104 bytes 2024-08-31T08:39:29.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1621136384 bytes 2024-08-31T08:39:29.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1706706944 bytes 2024-08-31T08:39:30.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1769331712 bytes 2024-08-31T08:39:30.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1806594048 bytes 2024-08-31T08:39:30.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1886644224 bytes 2024-08-31T08:39:30.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1924433920 bytes 2024-08-31T08:39:30.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1946824704 bytes 2024-08-31T08:39:30.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1983994880 bytes 2024-08-31T08:39:31.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2016401408 bytes 2024-08-31T08:39:31.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2094948352 bytes 2024-08-31T08:39:31.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2155094016 bytes 2024-08-31T08:39:31.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_49 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:39:31.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_49 128M 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:39:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:39:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:39:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:31.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_49 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:39:31.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:31.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2024-08-31T08:39:31.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:39:31.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2024-08-31T08:39:31.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_49': 2024-08-31T08:39:31.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1493941d58a7 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1493941d58a7 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:31 2024 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:31 2024 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:31 2024 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:39:31.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:39:31.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b080f365-a5bd-4cee-9717-8d5cc278ced7 2024-08-31T08:39:31.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:39:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:39:31.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_49 2024-08-31T08:39:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:39:31.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:39:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:39:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:39:31.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:39:32.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 38755328 bytes 2024-08-31T08:39:32.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 88444928 bytes 2024-08-31T08:39:32.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 119946240 bytes 2024-08-31T08:39:32.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 215903232 bytes 2024-08-31T08:39:33.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 296597504 bytes 2024-08-31T08:39:33.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 319480832 bytes 2024-08-31T08:39:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 392860672 bytes 2024-08-31T08:39:33.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 424919040 bytes 2024-08-31T08:39:33.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 456177664 bytes 2024-08-31T08:39:33.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 489745408 bytes 2024-08-31T08:39:34.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 508916736 bytes 2024-08-31T08:39:34.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 541881344 bytes 2024-08-31T08:39:34.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 582727680 bytes 2024-08-31T08:39:34.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 641516544 bytes 2024-08-31T08:39:34.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 664026112 bytes 2024-08-31T08:39:35.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 737457152 bytes 2024-08-31T08:39:35.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 779772928 bytes 2024-08-31T08:39:35.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 898109440 bytes 2024-08-31T08:39:35.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 919885824 bytes 2024-08-31T08:39:35.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 946397184 bytes 2024-08-31T08:39:35.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 961617920 bytes 2024-08-31T08:39:36.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 974501888 bytes 2024-08-31T08:39:36.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1000844288 bytes 2024-08-31T08:39:36.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1043136512 bytes 2024-08-31T08:39:36.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1074757632 bytes 2024-08-31T08:39:36.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1097957376 bytes 2024-08-31T08:39:36.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1155322880 bytes 2024-08-31T08:39:37.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1215712256 bytes 2024-08-31T08:39:37.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1278990336 bytes 2024-08-31T08:39:37.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1303029760 bytes 2024-08-31T08:39:37.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1326238720 bytes 2024-08-31T08:39:37.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1402993664 bytes 2024-08-31T08:39:37.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1452934144 bytes 2024-08-31T08:39:38.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1500028928 bytes 2024-08-31T08:39:38.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1573174272 bytes 2024-08-31T08:39:38.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1634791424 bytes 2024-08-31T08:39:38.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1658139648 bytes 2024-08-31T08:39:38.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1676219392 bytes 2024-08-31T08:39:39.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1692748800 bytes 2024-08-31T08:39:39.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1701582848 bytes 2024-08-31T08:39:39.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1751591936 bytes 2024-08-31T08:39:39.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1775281152 bytes 2024-08-31T08:39:39.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1837245440 bytes 2024-08-31T08:39:39.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1952987136 bytes 2024-08-31T08:39:40.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2000113664 bytes 2024-08-31T08:39:40.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2030885888 bytes 2024-08-31T08:39:40.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2049192960 bytes 2024-08-31T08:39:40.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2075245568 bytes 2024-08-31T08:39:40.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2079749120 bytes 2024-08-31T08:39:40.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2091705344 bytes 2024-08-31T08:39:40.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2138972160 bytes 2024-08-31T08:39:41.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2157833216 bytes 2024-08-31T08:39:41.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2228077568 bytes 2024-08-31T08:39:41.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2269416448 bytes 2024-08-31T08:39:41.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2337163264 bytes 2024-08-31T08:39:41.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2416746496 bytes 2024-08-31T08:39:42.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2456256512 bytes 2024-08-31T08:39:42.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2467895296 bytes 2024-08-31T08:39:42.380 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2468538368 bytes 2024-08-31T08:39:42.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2521530368 bytes 2024-08-31T08:39:42.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2582953984 bytes 2024-08-31T08:39:42.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2631934976 bytes 2024-08-31T08:39:43.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2704887808 bytes 2024-08-31T08:39:43.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2784462848 bytes 2024-08-31T08:39:43.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2855308288 bytes 2024-08-31T08:39:43.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2938439680 bytes 2024-08-31T08:39:44.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2941176832 bytes 2024-08-31T08:39:44.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 3015240704 bytes 2024-08-31T08:39:44.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 3158367232 bytes 2024-08-31T08:39:44.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3185285120 bytes 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_50 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2024-08-31T08:39:44.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n snapshot ']' 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_50 128M 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:44.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 128M ']' 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=128M 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:39:44.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_50 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ is_pool_mirror_mode_image mirror 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' image = image ']' 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:39:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2024-08-31T08:39:45.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:39:45.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2024-08-31T08:39:45.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_50': 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 149c93f86745 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.149c93f86745 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:44 2024 2024-08-31T08:39:45.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:44 2024 2024-08-31T08:39:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:44 2024 2024-08-31T08:39:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:39:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:39:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1fd95124-6f6d-4ddf-a6ce-7b0aec68c31f 2024-08-31T08:39:45.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:39:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n '' ']' 2024-08-31T08:39:45.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_50 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=12 2024-08-31T08:39:45.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:39:45.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:39:45.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 25761792 bytes 2024-08-31T08:39:46.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 81215488 bytes 2024-08-31T08:39:46.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 103950336 bytes 2024-08-31T08:39:46.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 177872896 bytes 2024-08-31T08:39:46.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 247201792 bytes 2024-08-31T08:39:47.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 333298688 bytes 2024-08-31T08:39:47.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 340360192 bytes 2024-08-31T08:39:47.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 385557504 bytes 2024-08-31T08:39:47.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 399498240 bytes 2024-08-31T08:39:47.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 468241408 bytes 2024-08-31T08:39:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 509050880 bytes 2024-08-31T08:39:48.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 556195840 bytes 2024-08-31T08:39:48.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 615147520 bytes 2024-08-31T08:39:48.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 628820992 bytes 2024-08-31T08:39:48.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 683282432 bytes 2024-08-31T08:39:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 728081408 bytes 2024-08-31T08:39:48.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 778095616 bytes 2024-08-31T08:39:49.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 795968512 bytes 2024-08-31T08:39:49.278 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 813389824 bytes 2024-08-31T08:39:49.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 834710528 bytes 2024-08-31T08:39:49.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 888406016 bytes 2024-08-31T08:39:49.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 959507456 bytes 2024-08-31T08:39:49.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 975482880 bytes 2024-08-31T08:39:50.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 992293888 bytes 2024-08-31T08:39:50.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1078854656 bytes 2024-08-31T08:39:50.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1104278528 bytes 2024-08-31T08:39:50.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1111786496 bytes 2024-08-31T08:39:50.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1135241216 bytes 2024-08-31T08:39:50.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1187325952 bytes 2024-08-31T08:39:51.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1194142720 bytes 2024-08-31T08:39:51.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1196841984 bytes 2024-08-31T08:39:51.257 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.32952568783476066) > weight (0.25) 2024-08-31T08:39:51.257 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7569401449634231) > weight (0.25) 2024-08-31T08:39:51.257 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-08-31T08:39:51.258 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-08-31T08:39:51.258 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:39:51.259 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:39:51.257+0000 7f35a3941640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 27042) UID: 1000 2024-08-31T08:39:51.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1269050368 bytes 2024-08-31T08:39:51.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1373550592 bytes 2024-08-31T08:39:51.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1398808576 bytes 2024-08-31T08:39:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1422415872 bytes 2024-08-31T08:39:52.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1442518016 bytes 2024-08-31T08:39:52.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1472283648 bytes 2024-08-31T08:39:52.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1512605696 bytes 2024-08-31T08:39:52.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1536288768 bytes 2024-08-31T08:39:52.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1595791360 bytes 2024-08-31T08:39:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1602400256 bytes 2024-08-31T08:39:53.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1623416832 bytes 2024-08-31T08:39:53.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1643701248 bytes 2024-08-31T08:39:53.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1697055744 bytes 2024-08-31T08:39:53.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1713678336 bytes 2024-08-31T08:39:53.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1744913408 bytes 2024-08-31T08:39:53.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1774308352 bytes 2024-08-31T08:39:54.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1838886912 bytes 2024-08-31T08:39:54.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1872422912 bytes 2024-08-31T08:39:54.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1873036288 bytes 2024-08-31T08:39:54.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1893764096 bytes 2024-08-31T08:39:54.582 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 1942874112 bytes 2024-08-31T08:39:54.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 1986021376 bytes 2024-08-31T08:39:54.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 1999994880 bytes 2024-08-31T08:39:55.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2034340864 bytes 2024-08-31T08:39:55.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2111039488 bytes 2024-08-31T08:39:55.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2161461248 bytes 2024-08-31T08:39:55.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2195993600 bytes 2024-08-31T08:39:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2233409536 bytes 2024-08-31T08:39:56.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2272496640 bytes 2024-08-31T08:39:56.259 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-08-31T08:39:56.259 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:39:56.259 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-08-31T08:39:56.260 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-08-31T08:39:56.260 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-08-31T08:39:56.263 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-08-31T08:39:56.263 INFO:tasks.rbd_mirror_thrash:waiting for 111 secs before thrashing 2024-08-31T08:39:56.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2357908480 bytes 2024-08-31T08:39:56.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2434370560 bytes 2024-08-31T08:39:56.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2488449024 bytes 2024-08-31T08:39:56.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2501876736 bytes 2024-08-31T08:39:57.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2569228288 bytes 2024-08-31T08:39:57.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2594900992 bytes 2024-08-31T08:39:57.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2619770880 bytes 2024-08-31T08:39:57.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2688371712 bytes 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_images cluster2 mirror 50 2024-08-31T08:39:57.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_count=50 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local count 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_count=0 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:39:57.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:39:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:39:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:39:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-08-31T08:39:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'images: ' 2024-08-31T08:39:57.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 2 2024-08-31T08:39:57.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:39:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ count=50 2024-08-31T08:39:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 50 = 50 2024-08-31T08:39:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_healthy cluster2 mirror 2024-08-31T08:39:57.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:57.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:57.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:39:57.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state 2024-08-31T08:39:57.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:39:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:39:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:39:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2024-08-31T08:39:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'image health:' 2024-08-31T08:39:57.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 3 2024-08-31T08:39:57.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:39:57.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=OK 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = ERROR 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = OK 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_images cluster1 mirror 50 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_count=50 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local count 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_count=0 2024-08-31T08:39:57.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:39:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:39:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:39:57.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:39:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-08-31T08:39:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'images: ' 2024-08-31T08:39:57.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 2 2024-08-31T08:39:57.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ count=50 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 50 = 50 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_healthy cluster1 mirror 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state 2024-08-31T08:39:57.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:39:57.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-08-31T08:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'image health:' 2024-08-31T08:39:57.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 3 2024-08-31T08:39:57.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:39:57.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=OK 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = ERROR 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = OK 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: compare many images' 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:39:57.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:39:57 TEST: compare many images 2024-08-31T08:39:57.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:39:57 TEST: compare many images 2024-08-31T08:39:57.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 50 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_1 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_1 snap 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:39:57.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:39:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2024-08-31T08:39:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:39:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:39:59.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:39:59.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:00.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2024-08-31T08:40:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2024-08-31T08:40:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:00.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:00.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:00.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:00.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:00.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:00.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:00.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:00.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:00.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2024-08-31T08:40:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2024-08-31T08:40:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:00.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:00.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:00.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:00.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:00.821 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:00.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:00.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:00.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:00.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2024-08-31T08:40:00.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:00.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:00.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:00.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:00.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_1 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_1 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_1.status 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_1.status 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:00.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.status 2024-08-31T08:40:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_1.status 2024-08-31T08:40:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_1.status 2024-08-31T08:40:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:00.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2024-08-31T08:40:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2024-08-31T08:40:01.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 86 2024-08-31T08:40:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.status 2024-08-31T08:40:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:01.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.status 2024-08-31T08:40:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2024-08-31T08:40:01.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:01.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:01.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=86 2024-08-31T08:40:01.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:01.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:01.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-08-31T08:40:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:02.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:02.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 86 = '' 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:02.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:02.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:02.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:02.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:02.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:02.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:02.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-08-31T08:40:02.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:02.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:02.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:02.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:02.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:02.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:02.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:02.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 86 = '' 2024-08-31T08:40:02.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:02.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-08-31T08:40:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:03.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 86 = '' 2024-08-31T08:40:03.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:03.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:04.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.status 2024-08-31T08:40:04.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2024-08-31T08:40:04.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:04.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=86 2024-08-31T08:40:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 86 = 86 2024-08-31T08:40:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2024-08-31T08:40:04.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:04.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:05.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2024-08-31T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_1': 2024-08-31T08:40:06.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11636f820b6f 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11636f820b6f 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:06.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:04 2024 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:04 2024 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:04 2024 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 7317f9a4-8aa2-4900-b19f-fff053c553f4 2024-08-31T08:40:06.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:06.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:06.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_1 snap 2024-08-31T08:40:06.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.export 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.export 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.export 2024-08-31T08:40:06.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.export 2024-08-31T08:40:06.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:06.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.export 2024-08-31T08:40:06.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:06.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.export 2024-08-31T08:40:06.518 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_1.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_1.export 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_2 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_2 snap 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:06.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:07.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:08.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:08.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2024-08-31T08:40:08.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:08.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:08.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:08.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:08.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:08.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:08.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:08.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:08.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:08.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2024-08-31T08:40:08.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2024-08-31T08:40:08.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:08.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:08.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:08.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:08.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:08.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:08.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:08.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:08.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2024-08-31T08:40:08.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:08.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:08.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_2 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:08.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_2 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_2.status 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_2.status 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.status 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_2.status 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_2.status 2024-08-31T08:40:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2024-08-31T08:40:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:08.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:08.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2024-08-31T08:40:09.349 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:40:09.349 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:40:09.350 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-08-31T08:40:09.350 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:40:09.349+0000 7f90adb6f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 39216) UID: 1000 2024-08-31T08:40:09.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 88 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.status 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.status 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2024-08-31T08:40:09.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:09.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=88 2024-08-31T08:40:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:09.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:09.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:09.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:09.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:09.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:09.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:09.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-08-31T08:40:09.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:10.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:10.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:10.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 88 = '' 2024-08-31T08:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:10.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-08-31T08:40:10.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:10.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:10.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:10.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:10.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:10.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:10.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:10.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 88 = '' 2024-08-31T08:40:10.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:10.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:11.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:11.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-08-31T08:40:11.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:11.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 88 = '' 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:11.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:11.350 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:40:11.350 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:40:11.351 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-08-31T08:40:11.351 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:40:11.351 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:40:11.353 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:40:11.354 INFO:tasks.rbd_mirror_thrash:waiting for 116 secs before thrashing 2024-08-31T08:40:12.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:12.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:12.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:12.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.status 2024-08-31T08:40:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2024-08-31T08:40:12.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:12.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:12.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=88 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 88 = 88 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:13.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_2': 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163c593ae24 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163c593ae24 2024-08-31T08:40:14.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:12 2024 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:12 2024 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:12 2024 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:14.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b9749456-99a0-4f9d-86ad-1654ff6a6a36 2024-08-31T08:40:14.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_2 snap 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.export 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.export 2024-08-31T08:40:14.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.export 2024-08-31T08:40:14.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.export 2024-08-31T08:40:14.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:14.255 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.export 2024-08-31T08:40:14.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:14.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.export 2024-08-31T08:40:14.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_2.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_2.export 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_3 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_3 snap 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:14.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:14.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:14.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:15.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:15.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:16.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2024-08-31T08:40:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:16.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:16.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:16.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:16.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:16.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:16.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:16.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2024-08-31T08:40:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2024-08-31T08:40:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:16.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:16.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2024-08-31T08:40:17.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:17.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:17.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:17.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:17.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_3 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:17.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_3 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_3.status 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_3.status 2024-08-31T08:40:17.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.status 2024-08-31T08:40:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_3.status 2024-08-31T08:40:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_3.status 2024-08-31T08:40:17.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:17.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2024-08-31T08:40:17.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 90 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.status 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.status 2024-08-31T08:40:17.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2024-08-31T08:40:17.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:17.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:17.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=90 2024-08-31T08:40:17.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:17.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:17.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-08-31T08:40:17.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:18.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:18.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 90 = '' 2024-08-31T08:40:18.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:18.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:18.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-08-31T08:40:18.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:18.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:18.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:18.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:18.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:18.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 90 = '' 2024-08-31T08:40:18.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:18.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-08-31T08:40:19.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:19.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:19.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:19.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 90 = '' 2024-08-31T08:40:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:19.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:20.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:20.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:20.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.status 2024-08-31T08:40:20.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2024-08-31T08:40:20.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:21.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=90 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 90 = 90 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:21.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:21.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:22.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2024-08-31T08:40:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_3': 2024-08-31T08:40:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:22.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11632e30afaa 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11632e30afaa 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:25 2024 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:25 2024 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:25 2024 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c6165d76-3f8e-4fae-95c8-1c9ff3d6281d 2024-08-31T08:40:22.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:22.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:22.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_3 snap 2024-08-31T08:40:22.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:22.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:40:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.export 2024-08-31T08:40:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.export 2024-08-31T08:40:22.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.export 2024-08-31T08:40:22.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.export 2024-08-31T08:40:22.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:22.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.export 2024-08-31T08:40:22.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:22.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.export 2024-08-31T08:40:22.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_3.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_3.export 2024-08-31T08:40:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_4 2024-08-31T08:40:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_4 snap 2024-08-31T08:40:22.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:22.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:23.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2024-08-31T08:40:23.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:23.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2024-08-31T08:40:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:24.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:24.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:24.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:24.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:24.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:24.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:24.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:24.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:24.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:24.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:24.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:24.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2024-08-31T08:40:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2024-08-31T08:40:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:24.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2024-08-31T08:40:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:24.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:24.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:24.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2024-08-31T08:40:24.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:24.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:24.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:24.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:24.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_4 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:25.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_4 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_4.status 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_4.status 2024-08-31T08:40:25.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:25.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.status 2024-08-31T08:40:25.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_4.status 2024-08-31T08:40:25.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_4.status 2024-08-31T08:40:25.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:25.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2024-08-31T08:40:25.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 92 2024-08-31T08:40:25.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.status 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.status 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2024-08-31T08:40:25.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:25.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=92 2024-08-31T08:40:25.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:25.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:25.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:26.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:26.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:26.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:26.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:26.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:26.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-08-31T08:40:26.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:26.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:26.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 92 = '' 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:26.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:26.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:26.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:26.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:26.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:26.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:26.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-08-31T08:40:26.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:26.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:26.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:26.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:26.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:26.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:26.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:26.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 92 = '' 2024-08-31T08:40:26.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:26.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:27.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-08-31T08:40:27.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:27.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:27.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:27.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:27.360 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 92 = '' 2024-08-31T08:40:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:27.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.status 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2024-08-31T08:40:28.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:29.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=92 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 92 = 92 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:29.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:29.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:30.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2024-08-31T08:40:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_4': 2024-08-31T08:40:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:30.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163fa4729ea 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163fa4729ea 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:39 2024 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:39 2024 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:39 2024 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:30.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:30.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6227f895-18e3-43af-ab10-bcfa84946b75 2024-08-31T08:40:30.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_4 snap 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.export 2024-08-31T08:40:30.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.export 2024-08-31T08:40:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.export 2024-08-31T08:40:30.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.export 2024-08-31T08:40:30.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:30.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.export 2024-08-31T08:40:30.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:30.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.export 2024-08-31T08:40:30.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_4.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_4.export 2024-08-31T08:40:30.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_5 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_5 snap 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:30.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:31.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:31.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:32.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:32.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2024-08-31T08:40:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:32.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:32.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:32.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:32.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:32.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:32.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:32.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:32.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:32.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2024-08-31T08:40:32.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2024-08-31T08:40:32.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:32.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:32.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:32.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:32.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:32.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:32.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:32.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:32.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2024-08-31T08:40:33.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:33.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:33.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:33.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_5 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_5 2024-08-31T08:40:33.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_5.status 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_5.status 2024-08-31T08:40:33.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.status 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_5.status 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_5.status 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:33.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2024-08-31T08:40:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:33.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2024-08-31T08:40:33.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 94 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.status 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.status 2024-08-31T08:40:33.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2024-08-31T08:40:33.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:33.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=94 2024-08-31T08:40:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:33.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:34.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-08-31T08:40:34.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:34.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:34.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:34.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:34.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 94 = '' 2024-08-31T08:40:34.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:34.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-08-31T08:40:34.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:34.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:34.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 94 = '' 2024-08-31T08:40:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:35.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:35.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-08-31T08:40:35.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:35.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:35.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:35.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:35.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:35.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 94 = '' 2024-08-31T08:40:35.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:35.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:36.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.status 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2024-08-31T08:40:36.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=94 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 94 = 94 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:38.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_5': 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163335ef204 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163335ef204 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:38.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:31:53 2024 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:31:53 2024 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:31:53 2024 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 72388a70-106e-4eb6-a7f8-822bc6c40054 2024-08-31T08:40:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_5 snap 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.export 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.export 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.export 2024-08-31T08:40:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.export 2024-08-31T08:40:38.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:38.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.export 2024-08-31T08:40:38.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:38.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.export 2024-08-31T08:40:38.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_5.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_5.export 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_6 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_6 snap 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:38.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2024-08-31T08:40:39.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:39.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:40.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:40.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:40.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:40.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:40.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:40.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:40.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:40.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:40.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:40.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:40.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:40.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:40.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2024-08-31T08:40:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2024-08-31T08:40:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:40.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:40.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:40.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:40.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:40.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2024-08-31T08:40:41.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:41.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:41.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:41.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:41.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:41.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_6 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_6 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:41.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:41.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:41.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:41.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_6.status 2024-08-31T08:40:41.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_6.status 2024-08-31T08:40:41.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.status 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_6.status 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_6.status 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:41.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2024-08-31T08:40:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:41.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2024-08-31T08:40:41.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 96 2024-08-31T08:40:41.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.status 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.status 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2024-08-31T08:40:41.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=96 2024-08-31T08:40:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:41.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:42.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:42.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:42.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:42.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:42.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:42.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-08-31T08:40:42.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:42.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:42.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=40 2024-08-31T08:40:42.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 96 = 40 2024-08-31T08:40:42.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:42.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-08-31T08:40:42.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:42.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:42.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=40 2024-08-31T08:40:42.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 96 = 40 2024-08-31T08:40:42.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:42.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:43.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-08-31T08:40:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 96 = '' 2024-08-31T08:40:43.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:43.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.status 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2024-08-31T08:40:45.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:45.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=96 2024-08-31T08:40:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 96 = 96 2024-08-31T08:40:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:45.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:46.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2024-08-31T08:40:46.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_6': 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11632a3a7b1a 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11632a3a7b1a 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:01 2024 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:01 2024 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:01 2024 2024-08-31T08:40:46.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 9f6764b3-49ac-434c-9bd1-5079abbddbf0 2024-08-31T08:40:46.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:46.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_6 snap 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.export 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.export 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.export 2024-08-31T08:40:46.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.export 2024-08-31T08:40:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:46.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.export 2024-08-31T08:40:46.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:46.596 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.export 2024-08-31T08:40:46.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_6.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_6.export 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_7 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_7 snap 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:46.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:47.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:47.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:48.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:48.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2024-08-31T08:40:48.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:48.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:48.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:48.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:48.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:48.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:48.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:48.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:48.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:48.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:48.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:48.820 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:48.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:48.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:48.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:48.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2024-08-31T08:40:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2024-08-31T08:40:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:48.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:48.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:48.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:48.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:48.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:48.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:48.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2024-08-31T08:40:49.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:49.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:49.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:49.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:49.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:49.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_7 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_7 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:49.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_7.status 2024-08-31T08:40:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_7.status 2024-08-31T08:40:49.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.status 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_7.status 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_7.status 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2024-08-31T08:40:49.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:49.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:49.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:49.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2024-08-31T08:40:49.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 98 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.status 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.status 2024-08-31T08:40:49.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2024-08-31T08:40:49.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:49.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=98 2024-08-31T08:40:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:49.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:50.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:50.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-08-31T08:40:50.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:50.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:50.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 98 = '' 2024-08-31T08:40:50.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:50.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:50.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:50.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-08-31T08:40:50.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:50.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:50.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 98 = '' 2024-08-31T08:40:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:50.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:51.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.status 2024-08-31T08:40:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2024-08-31T08:40:51.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:51.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=98 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 98 = 98 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:51.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:51.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:51.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:52.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2024-08-31T08:40:52.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_7': 2024-08-31T08:40:52.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11637216b55b 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11637216b55b 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:11 2024 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:11 2024 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:11 2024 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:52.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: be6ad6c7-3828-4fb6-ad59-75775c1277eb 2024-08-31T08:40:52.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_7 snap 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.export 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.export 2024-08-31T08:40:52.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.export 2024-08-31T08:40:52.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.export 2024-08-31T08:40:52.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:52.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.export 2024-08-31T08:40:52.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:52.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.export 2024-08-31T08:40:52.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_7.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_7.export 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_8 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_8 snap 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:53.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:54.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:40:54.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:54.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:54.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2024-08-31T08:40:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:55.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:40:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:40:55.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:40:55.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:55.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:55.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:40:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:40:55.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:40:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:40:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:40:55.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:40:55.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:40:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:40:55.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2024-08-31T08:40:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2024-08-31T08:40:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:40:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2024-08-31T08:40:55.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:40:55.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:40:55.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:40:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:40:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:40:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:40:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:40:55.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:40:55.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:55.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:55.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:40:55.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2024-08-31T08:40:56.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:40:56.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:40:56.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:40:56.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_8 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_8 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:40:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_8.status 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_8.status 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.status 2024-08-31T08:40:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_8.status 2024-08-31T08:40:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_8.status 2024-08-31T08:40:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:40:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:56.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2024-08-31T08:40:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:56.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2024-08-31T08:40:56.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 100 2024-08-31T08:40:56.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.status 2024-08-31T08:40:56.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:40:56.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:56.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:56.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.status 2024-08-31T08:40:56.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2024-08-31T08:40:56.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:56.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=100 2024-08-31T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:56.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:57.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-08-31T08:40:57.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:57.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 100 = '' 2024-08-31T08:40:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:57.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-08-31T08:40:57.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:40:57.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 100 = '' 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:40:57.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:40:58.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.status 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2024-08-31T08:40:58.391 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:40:58.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=100 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 100 = 100 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:40:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:40:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:40:59.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2024-08-31T08:40:59.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_8': 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116364cd57b7 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116364cd57b7 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:23 2024 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:23 2024 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:23 2024 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:40:59.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:40:59.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:40:59.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 29308e73-15d8-45be-8eab-74c2808c5487 2024-08-31T08:40:59.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:40:59.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:40:59.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_8 snap 2024-08-31T08:40:59.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.export 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.export 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.export 2024-08-31T08:40:59.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.export 2024-08-31T08:40:59.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:59.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.export 2024-08-31T08:40:59.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:40:59.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.export 2024-08-31T08:40:59.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_8.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_8.export 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_9 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_9 snap 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:00.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:01.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:01.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2024-08-31T08:41:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:02.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:02.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:02.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:02.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:02.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:02.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:02.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:02.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:02.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:02.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:02.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:02.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:02.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2024-08-31T08:41:02.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2024-08-31T08:41:02.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:02.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:02.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:02.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:02.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:02.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:02.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:02.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2024-08-31T08:41:03.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:03.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:03.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:03.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:03.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_9 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:03.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_9 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:03.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_9.status 2024-08-31T08:41:03.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_9.status 2024-08-31T08:41:03.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.status 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_9.status 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_9.status 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2024-08-31T08:41:03.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:03.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:03.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:03.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2024-08-31T08:41:03.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 102 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.status 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.status 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2024-08-31T08:41:03.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:03.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:03.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=102 2024-08-31T08:41:03.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:03.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:03.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:04.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-08-31T08:41:04.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:04.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:04.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:04.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 102 = '' 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:04.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:04.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-08-31T08:41:04.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:04.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 102 = '' 2024-08-31T08:41:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:04.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:05.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.status 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2024-08-31T08:41:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:05.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=102 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 102 = 102 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:05.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:06.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2024-08-31T08:41:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_9': 2024-08-31T08:41:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:06.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116387789da1 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116387789da1 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:30 2024 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:30 2024 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:30 2024 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:06.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:06.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: a832ab25-9838-4884-8321-006c701241dd 2024-08-31T08:41:06.493 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_9 snap 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.export 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.export 2024-08-31T08:41:06.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.export 2024-08-31T08:41:06.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.export 2024-08-31T08:41:06.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:06.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.export 2024-08-31T08:41:06.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:06.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.export 2024-08-31T08:41:06.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_9.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_9.export 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_10 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_10 snap 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:07.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:07.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:08.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:08.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2024-08-31T08:41:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:09.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:09.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:09.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:09.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2024-08-31T08:41:09.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:09.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:09.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:09.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:09.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:09.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2024-08-31T08:41:09.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2024-08-31T08:41:09.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:09.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:09.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:09.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:09.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:09.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:09.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:09.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:09.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:09.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2024-08-31T08:41:10.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:10.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:10.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:10.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:10.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:10.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_10 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_10 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:10.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_10.status 2024-08-31T08:41:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_10.status 2024-08-31T08:41:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:10.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.status 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_10.status 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_10.status 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:10.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:10.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2024-08-31T08:41:10.866 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.862+0000 7fb313f7e640 -1 reset not still connected to 0x55976983ab60 2024-08-31T08:41:10.870 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.868+0000 7fb313f7e640 -1 reset not still connected to 0x55976b86dad0 2024-08-31T08:41:10.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.874+0000 7fb313f7e640 -1 reset not still connected to 0x55976db9e410 2024-08-31T08:41:10.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.874+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d1860 2024-08-31T08:41:10.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 104 2024-08-31T08:41:10.881 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.878+0000 7fb313f7e640 -1 reset not still connected to 0x559781985110 2024-08-31T08:41:10.881 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.878+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdad0 2024-08-31T08:41:10.881 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.878+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdd40 2024-08-31T08:41:10.881 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:41:10.878+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fdee0 2024-08-31T08:41:10.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.status 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.status 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2024-08-31T08:41:10.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:10.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:10.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=104 2024-08-31T08:41:10.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:10.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:10.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:11.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-08-31T08:41:11.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:11.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:11.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:11.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:11.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:11.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:11.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:11.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 104 = '' 2024-08-31T08:41:11.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:11.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:11.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:11.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:11.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-08-31T08:41:11.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:11.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:11.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:11.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:11.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 104 = '' 2024-08-31T08:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:12.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.status 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2024-08-31T08:41:12.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:12.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=104 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 104 = 104 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:12.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:13.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2024-08-31T08:41:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_10': 2024-08-31T08:41:13.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163305700a1 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163305700a1 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:43 2024 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:43 2024 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:43 2024 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:13.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:13.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 138c0f6a-f5c2-4054-b5be-480015cc0759 2024-08-31T08:41:13.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_10 snap 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.export 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.export 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.export 2024-08-31T08:41:13.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.export 2024-08-31T08:41:13.744 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:13.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.export 2024-08-31T08:41:13.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:13.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.export 2024-08-31T08:41:13.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_10.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_10.export 2024-08-31T08:41:14.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_11 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_11 snap 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:14.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2024-08-31T08:41:15.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:15.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:16.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:16.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2024-08-31T08:41:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:16.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:16.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2024-08-31T08:41:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:16.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2024-08-31T08:41:16.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2024-08-31T08:41:16.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:16.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:16.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:16.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:16.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:17.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:17.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:17.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2024-08-31T08:41:17.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:17.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:17.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:17.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:17.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_11 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:17.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_11 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_11.status 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_11.status 2024-08-31T08:41:17.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.status 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_11.status 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_11.status 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:17.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2024-08-31T08:41:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:17.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2024-08-31T08:41:17.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 106 2024-08-31T08:41:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.status 2024-08-31T08:41:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:17.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.status 2024-08-31T08:41:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2024-08-31T08:41:17.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:17.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:17.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=106 2024-08-31T08:41:17.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:17.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:17.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:18.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-08-31T08:41:18.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:18.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:18.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:18.188 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:18.188 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 106 = '' 2024-08-31T08:41:18.188 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:18.188 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:18.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-08-31T08:41:18.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:18.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:18.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:18.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 106 = '' 2024-08-31T08:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:18.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:19.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:19.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:19.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:19.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:19.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.status 2024-08-31T08:41:19.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2024-08-31T08:41:19.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:19.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:19.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=106 2024-08-31T08:41:19.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 106 = 106 2024-08-31T08:41:19.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:19.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:19.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:20.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2024-08-31T08:41:20.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_11': 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163c75b2d64 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163c75b2d64 2024-08-31T08:41:20.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:32:50 2024 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:32:50 2024 2024-08-31T08:41:20.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:32:50 2024 2024-08-31T08:41:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 543fba89-43e8-4a84-a10b-ed65255deb74 2024-08-31T08:41:20.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:20.537 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_11 snap 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.export 2024-08-31T08:41:20.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.export 2024-08-31T08:41:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.export 2024-08-31T08:41:20.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.export 2024-08-31T08:41:20.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.export 2024-08-31T08:41:20.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:20.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.export 2024-08-31T08:41:21.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_11.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_11.export 2024-08-31T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_12 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_12 snap 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:21.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2024-08-31T08:41:22.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:22.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:23.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2024-08-31T08:41:23.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:23.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:23.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:23.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:23.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:23.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:23.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:23.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2024-08-31T08:41:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2024-08-31T08:41:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:23.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:23.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:23.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:23.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:23.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:23.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:23.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:23.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2024-08-31T08:41:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:23.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:23.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:23.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_12 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_12 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:23.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:23.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:23.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:23.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_12.status 2024-08-31T08:41:23.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_12.status 2024-08-31T08:41:23.473 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.status 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_12.status 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_12.status 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:23.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2024-08-31T08:41:23.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:23.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:23.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:23.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2024-08-31T08:41:24.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 108 2024-08-31T08:41:24.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.status 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.status 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2024-08-31T08:41:24.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:24.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:24.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=108 2024-08-31T08:41:24.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:24.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:24.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:24.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-08-31T08:41:24.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:24.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=46 2024-08-31T08:41:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 108 = 46 2024-08-31T08:41:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:24.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-08-31T08:41:24.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:25.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:25.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=46 2024-08-31T08:41:25.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 108 = 46 2024-08-31T08:41:25.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:25.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-08-31T08:41:25.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:25.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 108 = '' 2024-08-31T08:41:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:25.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:41:27.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.status 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2024-08-31T08:41:27.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=108 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 108 = 108 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:27.519 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:28.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2024-08-31T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_12': 2024-08-31T08:41:28.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163889889d5 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163889889d5 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:04 2024 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:04 2024 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:04 2024 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:28.567 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 32719a8b-c4e0-48d1-af14-d5d096fbbdaa 2024-08-31T08:41:28.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_12 snap 2024-08-31T08:41:28.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.export 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.export 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.export 2024-08-31T08:41:28.571 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.export 2024-08-31T08:41:28.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:28.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.export 2024-08-31T08:41:28.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:28.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.export 2024-08-31T08:41:29.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_12.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_12.export 2024-08-31T08:41:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_13 2024-08-31T08:41:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_13 snap 2024-08-31T08:41:29.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:29.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:30.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:30.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:31.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2024-08-31T08:41:31.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:31.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:31.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:31.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:31.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:31.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2024-08-31T08:41:31.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2024-08-31T08:41:31.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:31.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:31.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:31.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:31.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:31.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:31.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:31.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:31.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:31.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2024-08-31T08:41:32.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:32.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:32.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_13 2024-08-31T08:41:32.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_13 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_13.status 2024-08-31T08:41:32.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_13.status 2024-08-31T08:41:32.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.status 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_13.status 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_13.status 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:32.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2024-08-31T08:41:32.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 110 2024-08-31T08:41:32.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.status 2024-08-31T08:41:32.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.status 2024-08-31T08:41:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2024-08-31T08:41:32.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:32.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=110 2024-08-31T08:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:32.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:33.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-08-31T08:41:33.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:33.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:33.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:33.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 110 = '' 2024-08-31T08:41:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:33.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-08-31T08:41:33.556 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:33.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:33.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:33.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:33.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 110 = '' 2024-08-31T08:41:33.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:33.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:34.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:34.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-08-31T08:41:34.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:34.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:34.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 110 = '' 2024-08-31T08:41:34.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:34.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.status 2024-08-31T08:41:36.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2024-08-31T08:41:36.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:36.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=110 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 110 = 110 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:36.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:36.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:37.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_13': 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163d133e6fa 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163d133e6fa 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:37.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:19 2024 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:19 2024 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:19 2024 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 942a444b-45e7-45c0-b6f0-0f1f234cac1e 2024-08-31T08:41:37.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_13 snap 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:37.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.export 2024-08-31T08:41:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.export 2024-08-31T08:41:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.export 2024-08-31T08:41:37.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.export 2024-08-31T08:41:37.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:37.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.export 2024-08-31T08:41:37.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:37.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.export 2024-08-31T08:41:37.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_13.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_13.export 2024-08-31T08:41:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:37.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_14 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_14 snap 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:37.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:38.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:38.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2024-08-31T08:41:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:39.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:39.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:39.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:39.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:39.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2024-08-31T08:41:39.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:39.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:39.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:39.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:39.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:39.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:39.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:39.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2024-08-31T08:41:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2024-08-31T08:41:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:39.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:39.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:39.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:39.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2024-08-31T08:41:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:40.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_14 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:40.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_14 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_14.status 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_14.status 2024-08-31T08:41:40.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.status 2024-08-31T08:41:40.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_14.status 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_14.status 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:40.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2024-08-31T08:41:40.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 112 2024-08-31T08:41:40.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.status 2024-08-31T08:41:40.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:40.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:40.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.status 2024-08-31T08:41:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2024-08-31T08:41:40.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:40.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=112 2024-08-31T08:41:40.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:40.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:41.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:41.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-08-31T08:41:41.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:41.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:41.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:41.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:41.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:41.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:41.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 112 = '' 2024-08-31T08:41:41.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:41.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:41.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:41.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:41.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:41.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-08-31T08:41:41.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:41.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:41.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:41.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:41.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 112 = '' 2024-08-31T08:41:41.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:41.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:42.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.status 2024-08-31T08:41:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2024-08-31T08:41:42.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:42.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:42.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=112 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 112 = 112 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:42.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_14': 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116339a0fc55 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116339a0fc55 2024-08-31T08:41:43.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:28 2024 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:28 2024 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:28 2024 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1ef17613-0e21-45d7-b5d2-ab9520155a3c 2024-08-31T08:41:43.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_14 snap 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:43.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.export 2024-08-31T08:41:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.export 2024-08-31T08:41:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.export 2024-08-31T08:41:43.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.export 2024-08-31T08:41:43.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:43.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.export 2024-08-31T08:41:43.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:43.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.export 2024-08-31T08:41:44.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_14.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_14.export 2024-08-31T08:41:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_15 2024-08-31T08:41:44.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_15 snap 2024-08-31T08:41:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:44.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:45.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:45.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:45.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:46.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2024-08-31T08:41:46.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:46.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2024-08-31T08:41:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:46.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:46.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:46.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:46.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:46.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:46.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2024-08-31T08:41:46.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2024-08-31T08:41:46.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:46.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2024-08-31T08:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:46.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:46.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:46.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:46.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:46.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:46.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2024-08-31T08:41:47.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:47.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_15 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:47.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_15 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_15.status 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_15.status 2024-08-31T08:41:47.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.status 2024-08-31T08:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_15.status 2024-08-31T08:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_15.status 2024-08-31T08:41:47.115 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:47.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2024-08-31T08:41:47.264 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5002720490188178) > weight (0.25) 2024-08-31T08:41:47.264 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:41:47.264 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:41:47.264 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:41:47.268 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:41:47.264+0000 7fedddc8b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 33898) UID: 1000 2024-08-31T08:41:47.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 114 2024-08-31T08:41:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.status 2024-08-31T08:41:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:47.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:47.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.status 2024-08-31T08:41:47.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2024-08-31T08:41:47.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:47.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=114 2024-08-31T08:41:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:47.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:47.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-08-31T08:41:48.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:48.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:48.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 114 = '' 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:48.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:48.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-08-31T08:41:48.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:48.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:48.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:48.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:48.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:48.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:48.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 114 = '' 2024-08-31T08:41:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:48.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:49.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-08-31T08:41:49.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:49.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:49.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:49.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:49.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:49.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:49.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:49.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 114 = '' 2024-08-31T08:41:49.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:49.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.status 2024-08-31T08:41:51.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2024-08-31T08:41:51.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:51.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=114 2024-08-31T08:41:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 114 = 114 2024-08-31T08:41:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2024-08-31T08:41:51.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:51.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:52.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2024-08-31T08:41:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_15': 2024-08-31T08:41:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:41:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:41:52.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11635797007e 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11635797007e 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:41:52.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:41:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:35 2024 2024-08-31T08:41:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:35 2024 2024-08-31T08:41:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:35 2024 2024-08-31T08:41:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:41:52.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:41:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:41:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: ff694f40-7359-4371-9cac-7bf8444def68 2024-08-31T08:41:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:41:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_15 snap 2024-08-31T08:41:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:41:52.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:52.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:41:52.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.export 2024-08-31T08:41:52.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.export 2024-08-31T08:41:52.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.export 2024-08-31T08:41:52.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.export 2024-08-31T08:41:52.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:52.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.export 2024-08-31T08:41:52.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:41:52.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.export 2024-08-31T08:41:52.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_15.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_15.export 2024-08-31T08:41:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:41:52.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_16 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_16 snap 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:52.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:41:53.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:53.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:41:54.266 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:41:54.266 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:41:54.266 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:41:54.266 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:41:54.267 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:41:54.269 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:41:54.270 INFO:tasks.rbd_mirror_thrash:waiting for 79 secs before thrashing 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:41:54.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:41:54.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:41:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:41:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:41:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:41:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:41:54.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:41:54.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:54.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2024-08-31T08:41:54.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:41:54.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:54.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:41:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:41:54.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:41:54.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:41:54.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2024-08-31T08:41:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2024-08-31T08:41:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:41:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:41:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:41:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2024-08-31T08:41:54.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:41:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:41:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:41:54.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:41:54.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:41:54.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:41:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:41:55.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2024-08-31T08:41:55.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:41:55.119 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:41:55.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:41:55.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:41:55.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_16 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_16 2024-08-31T08:41:55.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_16.status 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_16.status 2024-08-31T08:41:55.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.status 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_16.status 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_16.status 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2024-08-31T08:41:55.124 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:55.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2024-08-31T08:41:55.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 116 2024-08-31T08:41:55.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.status 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.status 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2024-08-31T08:41:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:55.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:41:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=116 2024-08-31T08:41:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:55.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:41:56.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:56.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:56.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:56.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:56.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:56.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-08-31T08:41:56.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:56.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 116 = '' 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:56.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:41:56.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-08-31T08:41:56.598 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:56.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:56.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:56.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:56.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:56.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:56.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:56.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 116 = '' 2024-08-31T08:41:56.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:56.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:41:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:57.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:57.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-08-31T08:41:57.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:57.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:41:57.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:41:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:41:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:41:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:41:57.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 116 = '' 2024-08-31T08:41:57.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:41:57.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:41:59.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.status 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2024-08-31T08:41:59.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:41:59.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:41:59.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=116 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 116 = 116 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:41:59.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:41:59.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:00.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2024-08-31T08:42:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_16': 2024-08-31T08:42:00.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163add63b0c 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163add63b0c 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:44 2024 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:44 2024 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:44 2024 2024-08-31T08:42:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 09ffa3ae-165e-4e02-b27b-6c2e5c0d8dff 2024-08-31T08:42:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_16 snap 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:00.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.export 2024-08-31T08:42:00.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.export 2024-08-31T08:42:00.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.export 2024-08-31T08:42:00.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.export 2024-08-31T08:42:00.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:00.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.export 2024-08-31T08:42:00.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:00.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.export 2024-08-31T08:42:00.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_16.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_16.export 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_17 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_17 snap 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:00.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:02.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:02.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:03.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2024-08-31T08:42:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:03.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:03.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:03.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:03.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:03.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:03.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2024-08-31T08:42:03.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2024-08-31T08:42:03.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:03.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:03.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:03.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:03.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:03.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2024-08-31T08:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:03.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:03.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_17 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_17 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:03.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_17.status 2024-08-31T08:42:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_17.status 2024-08-31T08:42:03.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.status 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_17.status 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_17.status 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2024-08-31T08:42:03.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:03.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2024-08-31T08:42:04.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 118 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.status 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.status 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2024-08-31T08:42:04.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:04.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:04.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=118 2024-08-31T08:42:04.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:04.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:04.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-08-31T08:42:04.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:04.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:04.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=51 2024-08-31T08:42:04.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 118 = 51 2024-08-31T08:42:04.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:04.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:05.297 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-08-31T08:42:05.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:05.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:05.350 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:05.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 118 = '' 2024-08-31T08:42:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:05.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-08-31T08:42:06.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:06.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:06.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:06.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 118 = '' 2024-08-31T08:42:06.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:06.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:07.354 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:42:07.355 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:42:07.355 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-08-31T08:42:07.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:42:07.354+0000 7f343b5a2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 40523) UID: 1000 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.status 2024-08-31T08:42:07.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2024-08-31T08:42:07.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:07.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=118 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 118 = 118 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:07.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:08.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_17': 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11635a724cbf 2024-08-31T08:42:08.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11635a724cbf 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:33:55 2024 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:33:55 2024 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:33:55 2024 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 188a5b44-125b-4376-a8fb-33c918538c19 2024-08-31T08:42:08.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:08.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:08.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_17 snap 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.export 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.export 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.export 2024-08-31T08:42:08.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.export 2024-08-31T08:42:09.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:09.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.export 2024-08-31T08:42:09.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:09.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.export 2024-08-31T08:42:09.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_17.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_17.export 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_18 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_18 snap 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:09.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:09.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:09.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:10.356 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:42:10.356 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:42:10.356 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2024-08-31T08:42:10.356 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2024-08-31T08:42:10.357 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2024-08-31T08:42:10.359 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2024-08-31T08:42:10.359 INFO:tasks.rbd_mirror_thrash:waiting for 70 secs before thrashing 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:11.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:11.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:11.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:11.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:11.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:11.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:11.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:11.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:11.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:11.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2024-08-31T08:42:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2024-08-31T08:42:11.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:11.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:11.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:11.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:11.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:11.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2024-08-31T08:42:11.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:11.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:11.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:11.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:11.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_18 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:11.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_18 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_18.status 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_18.status 2024-08-31T08:42:11.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.status 2024-08-31T08:42:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_18.status 2024-08-31T08:42:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_18.status 2024-08-31T08:42:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:11.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:11.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2024-08-31T08:42:11.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:11.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:11.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:11.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2024-08-31T08:42:12.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 120 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.status 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.status 2024-08-31T08:42:12.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2024-08-31T08:42:12.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:12.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:12.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=120 2024-08-31T08:42:12.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:12.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:12.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:12.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-08-31T08:42:12.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:12.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:12.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=52 2024-08-31T08:42:12.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 120 = 52 2024-08-31T08:42:12.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:12.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:13.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-08-31T08:42:13.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:13.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:13.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:13.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 120 = '' 2024-08-31T08:42:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:13.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-08-31T08:42:14.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:14.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:14.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:14.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:14.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:14.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:14.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 120 = '' 2024-08-31T08:42:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:14.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.status 2024-08-31T08:42:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2024-08-31T08:42:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:16.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:16.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=120 2024-08-31T08:42:16.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 120 = 120 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:16.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:17.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2024-08-31T08:42:17.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_18': 2024-08-31T08:42:17.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:17.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163b14032a3 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163b14032a3 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:03 2024 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:03 2024 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:03 2024 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:17.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:17.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:17.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 9210fc52-db1b-4553-ad9c-5982d4188f4f 2024-08-31T08:42:17.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_18 snap 2024-08-31T08:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:17.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.export 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.export 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.export 2024-08-31T08:42:17.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.export 2024-08-31T08:42:17.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:17.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.export 2024-08-31T08:42:17.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:17.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.export 2024-08-31T08:42:17.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_18.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_18.export 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_19 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_19 snap 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:17.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:17.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:19.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:19.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:20.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:20.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:20.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:20.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:20.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:20.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:20.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:20.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2024-08-31T08:42:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2024-08-31T08:42:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:20.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:20.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:20.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:20.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:20.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:20.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:20.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:20.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2024-08-31T08:42:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:20.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:20.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_19 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_19 2024-08-31T08:42:20.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_19.status 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_19.status 2024-08-31T08:42:20.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.status 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_19.status 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_19.status 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2024-08-31T08:42:20.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:20.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:20.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:20.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2024-08-31T08:42:21.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 122 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.status 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.status 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2024-08-31T08:42:21.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:21.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:21.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=122 2024-08-31T08:42:21.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:21.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:21.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-08-31T08:42:21.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:21.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:21.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=53 2024-08-31T08:42:21.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 122 = 53 2024-08-31T08:42:21.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:21.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:22.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-08-31T08:42:22.346 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:22.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:22.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:22.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 122 = '' 2024-08-31T08:42:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:22.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:23.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-08-31T08:42:23.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:23.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:23.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:23.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:23.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:23.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:23.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 122 = '' 2024-08-31T08:42:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:23.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:24.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.status 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2024-08-31T08:42:24.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:24.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:24.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=122 2024-08-31T08:42:24.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 122 = 122 2024-08-31T08:42:24.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:24.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:25.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2024-08-31T08:42:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_19': 2024-08-31T08:42:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:25.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116374c55b05 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116374c55b05 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:10 2024 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:10 2024 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:10 2024 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:25.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:25.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:25.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d6dd38ac-296d-4d45-a399-c21cb2856fd7 2024-08-31T08:42:25.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:25.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_19 snap 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.export 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.export 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.export 2024-08-31T08:42:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.export 2024-08-31T08:42:26.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:26.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.export 2024-08-31T08:42:26.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:26.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.export 2024-08-31T08:42:26.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_19.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_19.export 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_20 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_20 snap 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:26.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:27.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2024-08-31T08:42:27.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:27.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:28.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:28.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:28.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:28.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:28.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:28.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:28.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:28.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:28.646 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:28.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:28.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:28.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2024-08-31T08:42:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2024-08-31T08:42:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:28.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:28.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:28.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:28.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:28.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2024-08-31T08:42:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:28.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:28.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:28.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_20 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_20 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:28.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:28.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:28.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_20.status 2024-08-31T08:42:28.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_20.status 2024-08-31T08:42:28.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.status 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_20.status 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_20.status 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:28.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:28.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2024-08-31T08:42:29.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 124 2024-08-31T08:42:29.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.status 2024-08-31T08:42:29.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:29.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:29.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:29.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.status 2024-08-31T08:42:29.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2024-08-31T08:42:29.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:29.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:29.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=124 2024-08-31T08:42:29.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:29.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:29.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:29.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-08-31T08:42:29.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:29.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:29.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=54 2024-08-31T08:42:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 124 = 54 2024-08-31T08:42:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:29.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:30.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:30.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-08-31T08:42:30.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:30.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:30.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:30.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 124 = '' 2024-08-31T08:42:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:30.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-08-31T08:42:31.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:31.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:31.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:31.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:31.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:31.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 124 = '' 2024-08-31T08:42:31.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:31.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:32.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.status 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2024-08-31T08:42:32.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:32.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=124 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 124 = 124 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:32.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:32.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:33.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2024-08-31T08:42:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_20': 2024-08-31T08:42:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:33.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163ba31077c 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163ba31077c 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:23 2024 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:23 2024 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:23 2024 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 221ab01e-6802-4692-8a08-3ac552ff07a3 2024-08-31T08:42:33.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_20 snap 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.export 2024-08-31T08:42:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.export 2024-08-31T08:42:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.export 2024-08-31T08:42:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.export 2024-08-31T08:42:34.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:34.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.export 2024-08-31T08:42:34.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:34.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.export 2024-08-31T08:42:34.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_20.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_20.export 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_21 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_21 snap 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:34.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:34.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:35.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:35.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:35.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:35.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:35.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:36.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:36.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:36.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:36.668 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:36.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:36.671 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:36.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:36.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:36.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:36.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:36.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2024-08-31T08:42:36.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2024-08-31T08:42:36.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:36.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:36.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:36.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2024-08-31T08:42:36.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:36.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:36.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:36.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:36.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:36.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:36.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2024-08-31T08:42:36.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:36.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:36.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:36.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:36.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:36.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_21 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_21 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:36.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:36.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:36.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:36.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_21.status 2024-08-31T08:42:36.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_21.status 2024-08-31T08:42:36.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.status 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_21.status 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_21.status 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:36.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:36.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2024-08-31T08:42:37.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 126 2024-08-31T08:42:37.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.status 2024-08-31T08:42:37.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:37.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:37.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:37.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.status 2024-08-31T08:42:37.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2024-08-31T08:42:37.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:37.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=126 2024-08-31T08:42:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:37.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:37.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:37.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-08-31T08:42:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:37.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=55 2024-08-31T08:42:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 126 = 55 2024-08-31T08:42:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:37.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:38.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-08-31T08:42:38.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:38.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:38.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:38.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:38.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:38.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:38.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 126 = '' 2024-08-31T08:42:38.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:38.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:39.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:39.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:39.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:39.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:39.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.status 2024-08-31T08:42:39.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2024-08-31T08:42:39.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:39.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=126 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 126 = 126 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:39.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:39.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:40.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2024-08-31T08:42:40.530 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_21': 2024-08-31T08:42:40.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:40.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:40.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:40.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163ff92990 2024-08-31T08:42:40.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163ff92990 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:35 2024 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:35 2024 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:35 2024 2024-08-31T08:42:40.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:40.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:40.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:40.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 83a21b96-117e-4747-bd14-d641899f4663 2024-08-31T08:42:40.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_21 snap 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:40.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.export 2024-08-31T08:42:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.export 2024-08-31T08:42:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.export 2024-08-31T08:42:40.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.export 2024-08-31T08:42:40.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:40.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.export 2024-08-31T08:42:40.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:40.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.export 2024-08-31T08:42:41.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_21.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_21.export 2024-08-31T08:42:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_22 2024-08-31T08:42:41.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_22 snap 2024-08-31T08:42:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:41.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2024-08-31T08:42:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:42.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:42.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:43.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2024-08-31T08:42:43.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:43.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2024-08-31T08:42:43.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:43.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:43.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:43.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:43.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:43.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:43.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:43.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:43.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:43.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:43.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:43.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2024-08-31T08:42:43.783 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2024-08-31T08:42:43.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:43.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:43.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:43.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:43.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:43.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2024-08-31T08:42:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:43.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_22 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:43.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_22 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_22.status 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_22.status 2024-08-31T08:42:43.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.status 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_22.status 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_22.status 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:43.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2024-08-31T08:42:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:43.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2024-08-31T08:42:44.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 128 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.status 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.status 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2024-08-31T08:42:44.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:44.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:44.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=128 2024-08-31T08:42:44.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:44.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:44.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:44.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-08-31T08:42:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:44.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:44.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=56 2024-08-31T08:42:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 128 = 56 2024-08-31T08:42:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:44.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:45.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-08-31T08:42:45.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:45.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:45.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:45.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 128 = '' 2024-08-31T08:42:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:45.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:46.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:46.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:46.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:46.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:46.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:46.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-08-31T08:42:46.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:46.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:46.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:46.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 128 = '' 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:46.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.status 2024-08-31T08:42:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2024-08-31T08:42:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:47.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:47.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=128 2024-08-31T08:42:47.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 128 = 128 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:47.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:48.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_22': 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163aa6999d8 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163aa6999d8 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:49.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:34:49 2024 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:34:49 2024 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:34:49 2024 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 500dccbe-6d7c-4195-84c9-4bdc2b07ec24 2024-08-31T08:42:49.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_22 snap 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.export 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.export 2024-08-31T08:42:49.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.export 2024-08-31T08:42:49.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.export 2024-08-31T08:42:49.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:49.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.export 2024-08-31T08:42:49.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:49.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.export 2024-08-31T08:42:49.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_22.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_22.export 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_23 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_23 snap 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:49.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:51.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:51.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:51.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2024-08-31T08:42:51.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:51.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:51.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:51.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:51.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2024-08-31T08:42:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:51.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:51.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:51.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2024-08-31T08:42:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2024-08-31T08:42:51.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:51.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2024-08-31T08:42:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:51.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:51.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:51.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:51.810 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:51.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:51.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2024-08-31T08:42:51.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:42:51.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:42:51.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:42:51.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:42:51.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_23 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_23 2024-08-31T08:42:51.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_23.status 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_23.status 2024-08-31T08:42:51.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.status 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_23.status 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_23.status 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2024-08-31T08:42:51.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:51.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:51.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:51.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2024-08-31T08:42:52.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 130 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.status 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.status 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2024-08-31T08:42:52.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:52.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:42:52.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=130 2024-08-31T08:42:52.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:52.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-08-31T08:42:52.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:53.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=57 2024-08-31T08:42:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 130 = 57 2024-08-31T08:42:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:53.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:53.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-08-31T08:42:53.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:53.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:53.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 130 = '' 2024-08-31T08:42:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:53.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-08-31T08:42:54.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:54.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:42:54.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:54.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:42:54.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:42:54.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:42:54.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:42:54.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 130 = '' 2024-08-31T08:42:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:42:54.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.status 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2024-08-31T08:42:55.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:42:55.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=130 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 130 = 130 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:55.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:42:55.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:56.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_23': 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163bc16ccc4 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163bc16ccc4 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:42:57.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:04 2024 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:04 2024 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:04 2024 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d80f7262-f0bd-4606-9216-6d6e2b02c77a 2024-08-31T08:42:57.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_23 snap 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:42:57.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.export 2024-08-31T08:42:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.export 2024-08-31T08:42:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.export 2024-08-31T08:42:57.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.export 2024-08-31T08:42:57.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.export 2024-08-31T08:42:57.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:42:57.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.export 2024-08-31T08:42:57.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_23.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_23.export 2024-08-31T08:42:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:42:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:42:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_24 2024-08-31T08:42:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_24 snap 2024-08-31T08:42:57.550 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:42:57.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:57.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:42:57.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:42:57.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:42:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2024-08-31T08:42:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:42:59.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:42:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:42:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:42:59.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:42:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:42:59.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:59.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2024-08-31T08:42:59.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:42:59.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:59.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:42:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:42:59.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:42:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:42:59.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2024-08-31T08:42:59.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2024-08-31T08:42:59.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:42:59.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:42:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:42:59.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:42:59.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:42:59.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:42:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2024-08-31T08:43:00.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:00.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:00.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:00.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:00.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:00.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:00.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_24 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_24 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_24.status 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_24.status 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.status 2024-08-31T08:43:00.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_24.status 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_24.status 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2024-08-31T08:43:00.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 132 2024-08-31T08:43:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.status 2024-08-31T08:43:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:00.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:00.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:00.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.status 2024-08-31T08:43:00.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2024-08-31T08:43:00.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:00.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=132 2024-08-31T08:43:00.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:00.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:00.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-08-31T08:43:00.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:01.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:01.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:01.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 132 = '' 2024-08-31T08:43:01.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:01.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-08-31T08:43:01.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:01.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:01.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 132 = '' 2024-08-31T08:43:01.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:01.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:02.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-08-31T08:43:02.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:02.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:02.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:02.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:02.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 132 = '' 2024-08-31T08:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:02.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:03.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:03.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.status 2024-08-31T08:43:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2024-08-31T08:43:03.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:03.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:04.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=132 2024-08-31T08:43:04.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 132 = 132 2024-08-31T08:43:04.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:04.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:04.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:05.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_24': 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11634800d29e 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11634800d29e 2024-08-31T08:43:05.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:15 2024 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:15 2024 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:15 2024 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: fca44610-a28e-42e0-b629-6dcbe524a929 2024-08-31T08:43:05.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_24 snap 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.export 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.export 2024-08-31T08:43:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.export 2024-08-31T08:43:05.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.export 2024-08-31T08:43:05.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:05.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.export 2024-08-31T08:43:05.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:05.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.export 2024-08-31T08:43:05.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_24.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_24.export 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_25 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_25 snap 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:05.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2024-08-31T08:43:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:06.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:06.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:07.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:07.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:07.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:07.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:07.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:07.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:07.751 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:07.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:07.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:07.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:07.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:07.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2024-08-31T08:43:07.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2024-08-31T08:43:07.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:07.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:07.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:07.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:07.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2024-08-31T08:43:07.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_25 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_25 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_25.status 2024-08-31T08:43:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_25.status 2024-08-31T08:43:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.status 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_25.status 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_25.status 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2024-08-31T08:43:08.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 134 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.status 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.status 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2024-08-31T08:43:08.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:08.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=134 2024-08-31T08:43:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:08.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:09.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-08-31T08:43:09.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:09.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:09.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 134 = '' 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:09.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-08-31T08:43:09.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:09.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:09.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:09.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:09.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 134 = '' 2024-08-31T08:43:09.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:09.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:10.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-08-31T08:43:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:10.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:10.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:10.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:10.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 134 = '' 2024-08-31T08:43:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:10.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:11.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:11.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:11.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:11.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:11.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.status 2024-08-31T08:43:11.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2024-08-31T08:43:11.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:12.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:12.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=134 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 134 = 134 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:12.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:13.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2024-08-31T08:43:13.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_25': 2024-08-31T08:43:13.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:13.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163366c1b75 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163366c1b75 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:26 2024 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:26 2024 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:26 2024 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:13.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:13.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:13.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 8ee04712-4bc0-4a3d-a3bf-ac9627495068 2024-08-31T08:43:13.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:13.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:13.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_25 snap 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.export 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.export 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.export 2024-08-31T08:43:13.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.export 2024-08-31T08:43:13.271 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9837747911905677) > weight (0.25) 2024-08-31T08:43:13.271 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7864700652105939) > weight (0.25) 2024-08-31T08:43:13.271 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7335278651438015) > weight (0.25) 2024-08-31T08:43:13.271 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.42281065414167884) > weight (0.25) 2024-08-31T08:43:13.272 INFO:tasks.rbd_mirror_thrash:waiting for 62 secs before thrashing 2024-08-31T08:43:13.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:13.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.export 2024-08-31T08:43:13.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:13.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.export 2024-08-31T08:43:13.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_25.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_25.export 2024-08-31T08:43:13.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_26 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_26 snap 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:13.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:14.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:14.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:15.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:15.765 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:15.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:15.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:15.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:15.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:15.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:15.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2024-08-31T08:43:15.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2024-08-31T08:43:15.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:15.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:15.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:15.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:15.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:15.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:15.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:15.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2024-08-31T08:43:16.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:16.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_26 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:16.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_26 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_26.status 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_26.status 2024-08-31T08:43:16.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.status 2024-08-31T08:43:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_26.status 2024-08-31T08:43:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_26.status 2024-08-31T08:43:16.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:16.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2024-08-31T08:43:16.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 136 2024-08-31T08:43:16.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.status 2024-08-31T08:43:16.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:16.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:16.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:16.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.status 2024-08-31T08:43:16.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2024-08-31T08:43:16.786 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:16.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=136 2024-08-31T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:16.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:17.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:17.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:17.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:17.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:17.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:17.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-08-31T08:43:17.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:17.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:17.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:17.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:17.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:17.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:17.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:17.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 136 = '' 2024-08-31T08:43:17.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:17.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-08-31T08:43:17.492 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:17.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 136 = '' 2024-08-31T08:43:17.541 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:17.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:18.342 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-08-31T08:43:18.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:18.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:18.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:18.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 136 = '' 2024-08-31T08:43:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:18.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:20.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.status 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2024-08-31T08:43:20.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:20.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=136 2024-08-31T08:43:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 136 = 136 2024-08-31T08:43:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:20.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:20.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:20.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8656795759057815) > weight (0.25) 2024-08-31T08:43:20.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6756447262981505) > weight (0.25) 2024-08-31T08:43:20.360 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6288693124140871) > weight (0.25) 2024-08-31T08:43:20.360 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2024-08-31T08:43:20.361 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2024-08-31T08:43:20.361 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-08-31T08:43:20.362 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.smithi167.stderr:2024-08-31T08:43:20.361+0000 7fdf12b77640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 26983) UID: 1000 2024-08-31T08:43:21.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_26': 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11637219f494 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11637219f494 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:21.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:38 2024 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:38 2024 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:38 2024 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b5b3e638-f5aa-43cf-a2f1-98602c0046a0 2024-08-31T08:43:21.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_26 snap 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.export 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.export 2024-08-31T08:43:21.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.export 2024-08-31T08:43:21.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.export 2024-08-31T08:43:21.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:21.318 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.export 2024-08-31T08:43:21.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:21.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.export 2024-08-31T08:43:21.583 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_26.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_26.export 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_27 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_27 snap 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:21.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:21.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:21.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:21.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:22.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:22.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:22.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:23.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:23.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:23.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:23.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:23.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:23.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:23.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:23.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:23.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2024-08-31T08:43:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2024-08-31T08:43:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:23.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:23.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:23.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:23.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:23.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:23.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:23.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:23.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:23.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2024-08-31T08:43:24.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:24.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:24.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:24.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:24.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:24.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_27 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_27 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:24.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_27.status 2024-08-31T08:43:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_27.status 2024-08-31T08:43:24.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.status 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_27.status 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_27.status 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:24.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:24.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:24.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2024-08-31T08:43:24.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 138 2024-08-31T08:43:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.status 2024-08-31T08:43:24.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.status 2024-08-31T08:43:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2024-08-31T08:43:24.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:24.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:24.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=138 2024-08-31T08:43:24.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:24.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:24.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:25.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-08-31T08:43:25.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:25.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:25.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 138 = '' 2024-08-31T08:43:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:25.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-08-31T08:43:25.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:25.584 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:25.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:25.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:25.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:25.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:25.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 138 = '' 2024-08-31T08:43:25.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:25.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:26.327 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.325+0000 7fb313f7e640 -1 reset not still connected to 0x55976db9e410 2024-08-31T08:43:26.329 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.326+0000 7fb313f7e640 -1 reset not still connected to 0x559775196d00 2024-08-31T08:43:26.329 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.326+0000 7fb313f7e640 -1 reset not still connected to 0x55977c8a12b0 2024-08-31T08:43:26.329 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.326+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9b4f70 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9b52b0 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d1860 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x559781985110 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x559782750270 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fc8f0 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdad0 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597868fdd40 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x559795b389c0 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fc340 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fdee0 2024-08-31T08:43:26.331 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:43:26.328+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba7930 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-08-31T08:43:26.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:26.434 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:26.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:26.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:26.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:26.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:26.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:26.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 138 = '' 2024-08-31T08:43:26.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:26.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.status 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2024-08-31T08:43:28.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:28.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=138 2024-08-31T08:43:28.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 138 = 138 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:28.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:28.362 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2024-08-31T08:43:28.363 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:43:28.363 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2024-08-31T08:43:28.363 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2024-08-31T08:43:28.363 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2024-08-31T08:43:28.366 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2024-08-31T08:43:28.366 INFO:tasks.rbd_mirror_thrash:waiting for 71 secs before thrashing 2024-08-31T08:43:29.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_27': 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163a1e4155a 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163a1e4155a 2024-08-31T08:43:29.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:49 2024 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:49 2024 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:49 2024 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 23512681-0d1b-4424-93c6-c223988299d9 2024-08-31T08:43:29.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:29.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_27 snap 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.export 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.export 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.export 2024-08-31T08:43:29.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.export 2024-08-31T08:43:29.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:29.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.export 2024-08-31T08:43:29.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:29.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.export 2024-08-31T08:43:29.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_27.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_27.export 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_28 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_28 snap 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:29.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:30.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:30.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:30.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:30.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:30.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:30.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2024-08-31T08:43:31.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:31.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2024-08-31T08:43:31.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:31.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:31.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:31.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:31.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:31.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:31.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2024-08-31T08:43:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2024-08-31T08:43:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:31.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2024-08-31T08:43:31.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:31.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:31.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:31.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:31.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:31.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:31.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2024-08-31T08:43:32.094 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:32.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:32.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:32.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_28 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_28 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_28.status 2024-08-31T08:43:32.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_28.status 2024-08-31T08:43:32.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.status 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_28.status 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_28.status 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:32.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2024-08-31T08:43:32.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 140 2024-08-31T08:43:32.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.status 2024-08-31T08:43:32.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:32.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:32.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:32.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.status 2024-08-31T08:43:32.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2024-08-31T08:43:32.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:32.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:32.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=140 2024-08-31T08:43:32.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:32.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:32.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:33.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-08-31T08:43:33.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:33.166 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:33.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 140 = '' 2024-08-31T08:43:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:33.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:33.569 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-08-31T08:43:33.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:33.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 140 = '' 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:33.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:34.427 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-08-31T08:43:34.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:34.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:34.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:34.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 140 = '' 2024-08-31T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:34.490 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:36.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.status 2024-08-31T08:43:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2024-08-31T08:43:36.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:36.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=140 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 140 = 140 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:36.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:37.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2024-08-31T08:43:37.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_28': 2024-08-31T08:43:37.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163fde289d2 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163fde289d2 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:35:56 2024 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:35:56 2024 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:35:56 2024 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:37.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:37.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:37.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 32c899e0-aef6-4039-9156-ccc4a4e46708 2024-08-31T08:43:37.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_28 snap 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.export 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.export 2024-08-31T08:43:37.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.export 2024-08-31T08:43:37.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.export 2024-08-31T08:43:37.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:37.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.export 2024-08-31T08:43:37.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:37.824 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.export 2024-08-31T08:43:37.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_28.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_28.export 2024-08-31T08:43:37.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_29 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_29 snap 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:37.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:39.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2024-08-31T08:43:39.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:39.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:39.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:40.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:40.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:40.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:40.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:40.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:40.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2024-08-31T08:43:40.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:40.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:40.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:40.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:40.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:40.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:40.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2024-08-31T08:43:40.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2024-08-31T08:43:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:40.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:40.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:40.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:40.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:41.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:41.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2024-08-31T08:43:41.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:41.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:41.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_29 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:41.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_29 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_29.status 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_29.status 2024-08-31T08:43:41.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.status 2024-08-31T08:43:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_29.status 2024-08-31T08:43:41.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_29.status 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:41.138 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2024-08-31T08:43:41.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 142 2024-08-31T08:43:41.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.status 2024-08-31T08:43:41.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:41.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:41.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:41.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.status 2024-08-31T08:43:41.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2024-08-31T08:43:41.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:41.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:41.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=142 2024-08-31T08:43:41.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:41.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:41.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-08-31T08:43:42.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:42.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:42.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 142 = '' 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:42.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-08-31T08:43:42.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:42.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 142 = '' 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:42.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.status 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2024-08-31T08:43:43.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:43.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=142 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 142 = 142 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:43.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:43.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:43.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:43.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:43.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:44.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2024-08-31T08:43:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_29': 2024-08-31T08:43:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:44.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163c24f121a 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163c24f121a 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:07 2024 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:07 2024 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:07 2024 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:44.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b9fca043-4527-4ec0-8fb0-13a1a765c247 2024-08-31T08:43:44.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:44.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_29 snap 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.export 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.export 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.export 2024-08-31T08:43:44.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.export 2024-08-31T08:43:44.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:44.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.export 2024-08-31T08:43:45.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:45.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.export 2024-08-31T08:43:45.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_29.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_29.export 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_30 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_30 snap 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:45.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:45.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:46.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:46.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:47.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:47.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:47.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:47.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:47.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:47.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:47.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:47.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:47.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:47.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:47.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:47.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:47.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:47.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2024-08-31T08:43:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2024-08-31T08:43:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:48.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:48.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:48.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:48.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:48.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:48.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:48.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:48.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2024-08-31T08:43:48.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:48.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:48.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_30 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_30 2024-08-31T08:43:48.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_30.status 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_30.status 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.status 2024-08-31T08:43:48.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_30.status 2024-08-31T08:43:48.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_30.status 2024-08-31T08:43:48.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:48.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:48.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2024-08-31T08:43:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:48.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2024-08-31T08:43:48.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 144 2024-08-31T08:43:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.status 2024-08-31T08:43:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:48.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:48.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:48.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.status 2024-08-31T08:43:48.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2024-08-31T08:43:48.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:48.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=144 2024-08-31T08:43:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:48.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:49.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-08-31T08:43:49.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:49.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:49.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 144 = '' 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:49.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:49.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-08-31T08:43:49.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:49.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:49.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:49.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:49.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:49.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:49.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 144 = '' 2024-08-31T08:43:49.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:49.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:50.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.status 2024-08-31T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2024-08-31T08:43:50.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:50.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:50.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=144 2024-08-31T08:43:50.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 144 = 144 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:50.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:51.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_30': 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163dfba3938 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163dfba3938 2024-08-31T08:43:51.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:13 2024 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:13 2024 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:13 2024 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: f528088b-901a-428c-946e-f8201633ca46 2024-08-31T08:43:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_30 snap 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.export 2024-08-31T08:43:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.export 2024-08-31T08:43:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.export 2024-08-31T08:43:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.export 2024-08-31T08:43:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:51.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.export 2024-08-31T08:43:52.201 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:43:52.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.export 2024-08-31T08:43:52.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_30.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_30.export 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_31 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_31 snap 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:52.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:53.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:43:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:54.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2024-08-31T08:43:54.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:54.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:54.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:54.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:43:54.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:43:54.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:43:54.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:43:54.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:54.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:54.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:43:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:43:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:43:54.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:43:54.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:43:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:43:54.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2024-08-31T08:43:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2024-08-31T08:43:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:43:54.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:43:54.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:43:54.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:43:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:43:54.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2024-08-31T08:43:54.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:43:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:43:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:43:54.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:43:54.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_31 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:43:54.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_31 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_31.status 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_31.status 2024-08-31T08:43:54.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.status 2024-08-31T08:43:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_31.status 2024-08-31T08:43:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_31.status 2024-08-31T08:43:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:43:54.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2024-08-31T08:43:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:54.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2024-08-31T08:43:55.599 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 146 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.status 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.status 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2024-08-31T08:43:55.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:55.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:43:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=146 2024-08-31T08:43:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:55.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:43:55.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-08-31T08:43:55.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:55.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:55.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=65 2024-08-31T08:43:55.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 146 = 65 2024-08-31T08:43:55.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:55.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:43:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:56.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:56.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:56.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:56.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-08-31T08:43:56.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:56.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:56.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:56.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:56.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:56.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 146 = '' 2024-08-31T08:43:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:56.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-08-31T08:43:57.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:43:57.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:57.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:43:57.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:43:57.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:43:57.230 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 146 = '' 2024-08-31T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:43:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:43:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:58.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:58.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:58.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.status 2024-08-31T08:43:58.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2024-08-31T08:43:58.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:43:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:43:58.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=146 2024-08-31T08:43:58.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 146 = 146 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:43:58.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:43:59.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2024-08-31T08:43:59.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_31': 2024-08-31T08:43:59.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:43:59.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:43:59.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163d3191a6d 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163d3191a6d 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:23 2024 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:23 2024 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:23 2024 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:43:59.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:43:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 681647ab-f5ff-4fcf-8d6a-32f3fe7fa4ba 2024-08-31T08:43:59.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_31 snap 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:43:59.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:43:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.export 2024-08-31T08:43:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.export 2024-08-31T08:43:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.export 2024-08-31T08:43:59.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.export 2024-08-31T08:44:00.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:00.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.export 2024-08-31T08:44:00.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:00.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.export 2024-08-31T08:44:00.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_31.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_31.export 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_32 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_32 snap 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:00.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:00.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:00.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2024-08-31T08:44:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:01.619 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:01.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:02.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:02.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:02.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:02.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:02.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:02.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:02.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:02.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:02.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:02.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:02.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:02.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:02.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:02.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:02.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:02.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:02.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:02.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:02.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2024-08-31T08:44:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2024-08-31T08:44:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:02.727 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:02.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:02.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:02.732 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:02.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:02.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:02.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:02.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:02.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2024-08-31T08:44:02.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:02.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:02.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:02.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:02.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:02.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_32 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_32 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:02.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_32.status 2024-08-31T08:44:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_32.status 2024-08-31T08:44:02.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.status 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_32.status 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_32.status 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:02.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:02.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2024-08-31T08:44:03.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 148 2024-08-31T08:44:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.status 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.status 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2024-08-31T08:44:03.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:03.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:03.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=148 2024-08-31T08:44:03.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:03.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-08-31T08:44:03.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:03.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=66 2024-08-31T08:44:03.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 148 = 66 2024-08-31T08:44:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:03.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:04.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:04.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:04.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:04.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:04.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:04.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-08-31T08:44:04.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:04.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:04.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:04.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 148 = '' 2024-08-31T08:44:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:04.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-08-31T08:44:05.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:05.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:05.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:05.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:05.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:05.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:05.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:05.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 148 = '' 2024-08-31T08:44:05.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:05.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.status 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2024-08-31T08:44:06.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=148 2024-08-31T08:44:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 148 = 148 2024-08-31T08:44:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:06.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:06.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:07.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2024-08-31T08:44:07.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_32': 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116333e7511e 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116333e7511e 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:07.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:31 2024 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:31 2024 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:31 2024 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: f34b2723-009a-42b7-ad38-540c04928627 2024-08-31T08:44:07.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_32 snap 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.export 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.export 2024-08-31T08:44:07.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.export 2024-08-31T08:44:07.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.export 2024-08-31T08:44:08.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:08.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.export 2024-08-31T08:44:08.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:08.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.export 2024-08-31T08:44:08.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_32.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_32.export 2024-08-31T08:44:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:08.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_33 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_33 snap 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:08.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:09.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:09.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:09.855 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.853+0000 7f6cb0353640 -1 reset not still connected to 0x5599546be0d0 2024-08-31T08:44:09.855 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.853+0000 7f6cb0353640 -1 reset not still connected to 0x5599546beb60 2024-08-31T08:44:09.855 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.853+0000 7f6cb0353640 -1 reset not still connected to 0x5599546bfc70 2024-08-31T08:44:09.855 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.853+0000 7f6cb0353640 -1 reset not still connected to 0x559955341520 2024-08-31T08:44:09.860 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.858+0000 7f6cb0353640 -1 reset not still connected to 0x559955e83a00 2024-08-31T08:44:09.860 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:44:09.858+0000 7f6cb0353640 -1 reset not still connected to 0x55998272eb60 2024-08-31T08:44:10.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:10.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:10.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2024-08-31T08:44:10.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:10.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:10.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:10.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:10.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:10.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2024-08-31T08:44:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2024-08-31T08:44:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:10.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:10.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:10.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:10.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:10.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:10.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:10.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:10.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2024-08-31T08:44:10.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:10.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:10.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:10.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:10.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:10.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:10.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:10.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_33 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_33 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_33.status 2024-08-31T08:44:10.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_33.status 2024-08-31T08:44:10.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.status 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_33.status 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_33.status 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:10.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2024-08-31T08:44:11.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 150 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.status 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.status 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2024-08-31T08:44:11.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:11.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:11.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=150 2024-08-31T08:44:11.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:11.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:11.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:11.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:11.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:11.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:11.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:11.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:11.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-08-31T08:44:11.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:11.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=67 2024-08-31T08:44:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 150 = 67 2024-08-31T08:44:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:11.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-08-31T08:44:12.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:12.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 150 = '' 2024-08-31T08:44:12.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:12.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:13.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-08-31T08:44:13.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:13.259 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:13.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:13.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 150 = '' 2024-08-31T08:44:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:13.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.status 2024-08-31T08:44:14.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2024-08-31T08:44:14.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:14.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:14.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=150 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 150 = 150 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:14.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:15.273 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9277321788492634) > weight (0.25) 2024-08-31T08:44:15.273 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5031072311292519) > weight (0.25) 2024-08-31T08:44:15.273 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-08-31T08:44:15.273 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-08-31T08:44:15.273 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:44:15.274 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:44:15.273+0000 7fb2e20d9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 39926) UID: 1000 2024-08-31T08:44:15.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2024-08-31T08:44:15.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_33': 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11636b9f9853 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11636b9f9853 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:40 2024 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:40 2024 2024-08-31T08:44:15.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:40 2024 2024-08-31T08:44:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 88c939cc-d595-4585-959a-55085fcd0473 2024-08-31T08:44:15.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_33 snap 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.export 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.export 2024-08-31T08:44:15.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.export 2024-08-31T08:44:15.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.export 2024-08-31T08:44:16.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:16.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.export 2024-08-31T08:44:16.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:16.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.export 2024-08-31T08:44:16.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_33.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_33.export 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_34 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_34 snap 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:16.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:16.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:16.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:17.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2024-08-31T08:44:17.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:17.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:17.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:18.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2024-08-31T08:44:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:18.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:18.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:18.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:18.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:18.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:18.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:18.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:18.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:18.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:18.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2024-08-31T08:44:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2024-08-31T08:44:18.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:18.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:18.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:18.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:18.768 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:18.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:18.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2024-08-31T08:44:18.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:18.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:18.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:18.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:18.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:18.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_34 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_34 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:18.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_34.status 2024-08-31T08:44:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_34.status 2024-08-31T08:44:18.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.status 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_34.status 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_34.status 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:18.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2024-08-31T08:44:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:18.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2024-08-31T08:44:19.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 152 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.status 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.status 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2024-08-31T08:44:19.667 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:19.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=152 2024-08-31T08:44:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:19.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:19.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-08-31T08:44:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=68 2024-08-31T08:44:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 152 = 68 2024-08-31T08:44:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:20.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-08-31T08:44:20.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:20.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:20.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 152 = '' 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:20.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:21.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:21.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-08-31T08:44:21.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:21.273 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:21.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:21.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:21.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 152 = '' 2024-08-31T08:44:21.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:21.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:22.274 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-08-31T08:44:22.274 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:44:22.274 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2024-08-31T08:44:22.274 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2024-08-31T08:44:22.275 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2024-08-31T08:44:22.278 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2024-08-31T08:44:22.278 INFO:tasks.rbd_mirror_thrash:waiting for 69 secs before thrashing 2024-08-31T08:44:22.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:22.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:22.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:22.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:22.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.status 2024-08-31T08:44:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2024-08-31T08:44:22.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:22.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=152 2024-08-31T08:44:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 152 = 152 2024-08-31T08:44:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2024-08-31T08:44:22.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:22.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:23.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_34': 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11634ef3cc3 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11634ef3cc3 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:23.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:36:51 2024 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:36:51 2024 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:36:51 2024 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d946a5bb-2842-4293-bff6-1ff598883e15 2024-08-31T08:44:23.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:23.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_34 snap 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.export 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.export 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.export 2024-08-31T08:44:23.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.export 2024-08-31T08:44:24.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:24.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.export 2024-08-31T08:44:24.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:24.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.export 2024-08-31T08:44:24.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_34.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_34.export 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_35 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_35 snap 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:24.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:25.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:25.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:26.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:26.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:26.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:26.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2024-08-31T08:44:26.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:26.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:26.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:26.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:26.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2024-08-31T08:44:26.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2024-08-31T08:44:26.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:26.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:26.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:26.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:26.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:26.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:26.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2024-08-31T08:44:26.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:26.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:26.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:26.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_35 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_35 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:26.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_35.status 2024-08-31T08:44:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_35.status 2024-08-31T08:44:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.status 2024-08-31T08:44:26.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_35.status 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_35.status 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:26.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2024-08-31T08:44:27.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 154 2024-08-31T08:44:27.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.status 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.status 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2024-08-31T08:44:27.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:27.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=154 2024-08-31T08:44:27.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:27.735 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:27.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-08-31T08:44:27.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:27.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:27.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=69 2024-08-31T08:44:27.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 154 = 69 2024-08-31T08:44:27.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:27.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:28.387 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-08-31T08:44:28.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:28.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:28.436 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:28.437 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 154 = '' 2024-08-31T08:44:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:28.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:29.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:29.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-08-31T08:44:29.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:29.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:29.334 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:29.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 154 = '' 2024-08-31T08:44:29.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:29.337 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:30.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.status 2024-08-31T08:44:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2024-08-31T08:44:30.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:30.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:30.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=154 2024-08-31T08:44:30.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 154 = 154 2024-08-31T08:44:30.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:30.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:31.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_35': 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11637b585ed4 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11637b585ed4 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:32.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:04 2024 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:04 2024 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:04 2024 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c96b3a90-a801-484f-ba1a-ada1712c4cb4 2024-08-31T08:44:32.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:32.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_35 snap 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.export 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.export 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.export 2024-08-31T08:44:32.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.export 2024-08-31T08:44:32.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.export 2024-08-31T08:44:32.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:32.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.export 2024-08-31T08:44:32.557 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_35.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_35.export 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_36 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_36 snap 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:32.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:32.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:33.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:34.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:34.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:34.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:34.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:34.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:34.695 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:34.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:34.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:34.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2024-08-31T08:44:34.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2024-08-31T08:44:34.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:34.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:34.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:34.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:34.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:34.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:34.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:34.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:34.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2024-08-31T08:44:34.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:34.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:34.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:34.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:34.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_36 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:34.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_36 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_36.status 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_36.status 2024-08-31T08:44:34.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.status 2024-08-31T08:44:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_36.status 2024-08-31T08:44:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_36.status 2024-08-31T08:44:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:34.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2024-08-31T08:44:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:34.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2024-08-31T08:44:35.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 156 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.status 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.status 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2024-08-31T08:44:35.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:35.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=156 2024-08-31T08:44:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:35.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:35.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:35.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-08-31T08:44:35.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:35.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=70 2024-08-31T08:44:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 156 = 70 2024-08-31T08:44:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:35.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:36.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-08-31T08:44:36.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:36.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:36.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:36.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 156 = '' 2024-08-31T08:44:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:36.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-08-31T08:44:37.245 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:37.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:37.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:37.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:37.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:37.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:37.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:37.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 156 = '' 2024-08-31T08:44:37.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:37.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.status 2024-08-31T08:44:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2024-08-31T08:44:38.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:38.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=156 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 156 = 156 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:38.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:38.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:38.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:38.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:38.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:39.367 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9772357519355561) > weight (0.25) 2024-08-31T08:44:39.367 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-08-31T08:44:39.367 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-08-31T08:44:39.367 INFO:tasks.rbd_mirror_thrash:waiting for 3 secs before reviving daemons 2024-08-31T08:44:39.368 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:44:39.367+0000 7f9e5820d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 30462) UID: 1000 2024-08-31T08:44:39.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2024-08-31T08:44:40.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_36': 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163a669b308 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163a669b308 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:40.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:14 2024 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:14 2024 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:14 2024 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 7c94795d-bc10-44e4-9ddb-b555399db9b4 2024-08-31T08:44:40.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:40.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:40.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_36 snap 2024-08-31T08:44:40.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:40.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.export 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.export 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.export 2024-08-31T08:44:40.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.export 2024-08-31T08:44:40.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:40.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.export 2024-08-31T08:44:40.495 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:40.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.export 2024-08-31T08:44:40.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_36.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_36.export 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_37 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_37 snap 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:40.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:41.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:41.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:42.368 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-08-31T08:44:42.369 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:44:42.369 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-08-31T08:44:42.369 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:44:42.369 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:44:42.372 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:44:42.372 INFO:tasks.rbd_mirror_thrash:waiting for 100 secs before thrashing 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:42.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:42.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2024-08-31T08:44:42.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:42.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:42.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:42.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:42.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:42.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:42.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:42.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:42.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2024-08-31T08:44:42.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:42.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:42.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:42.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:42.801 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:42.802 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:42.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:42.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2024-08-31T08:44:42.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:42.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:42.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_37 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:42.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:44:42.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:44:42.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:44:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:42.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_37 2024-08-31T08:44:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:42.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:42.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:42.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_37 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:3 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:3 cluster instance 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:3 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=3 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 3 = cluster1:3 ']' 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:42.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=3 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:3 cluster1 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:3 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:3 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=3 2024-08-31T08:44:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 3 = cluster1:3 ']' 2024-08-31T08:44:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=3 2024-08-31T08:44:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=3 2024-08-31T08:44:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2024-08-31T08:44:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:44:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:44:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2024-08-31T08:44:43.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_37 2024-08-31T08:44:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:43.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:43.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 1 2024-08-31T08:44:44.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:44.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:44.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:44.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:44.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:44.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:44.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:44.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:44.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2024-08-31T08:44:44.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:44.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:44.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:44.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:44.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:44.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:44.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2024-08-31T08:44:44.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2024-08-31T08:44:44.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:44.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_37 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:44.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:44:44.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:44:44.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:44:44.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:44.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:44.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:44.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_37 2024-08-31T08:44:44.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:44.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_37 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:44.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_37 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:44.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_37.status 2024-08-31T08:44:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_37.status 2024-08-31T08:44:44.417 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.status 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_37.status 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_37.status 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2024-08-31T08:44:44.418 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:44.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:44.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:44.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2024-08-31T08:44:44.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 158 2024-08-31T08:44:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.status 2024-08-31T08:44:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.status 2024-08-31T08:44:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2024-08-31T08:44:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:44.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:44.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=158 2024-08-31T08:44:44.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:44.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:44.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:44.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-08-31T08:44:44.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:45.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:45.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=71 2024-08-31T08:44:45.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 158 = 71 2024-08-31T08:44:45.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:45.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-08-31T08:44:45.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:45.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:45.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:45.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:45.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 158 = '' 2024-08-31T08:44:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:45.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:46.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-08-31T08:44:46.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:46.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:46.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:46.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 158 = '' 2024-08-31T08:44:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:46.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.status 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2024-08-31T08:44:47.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:47.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:47.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=158 2024-08-31T08:44:47.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 158 = 158 2024-08-31T08:44:47.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:47.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:48.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_37': 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11638e660bea 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11638e660bea 2024-08-31T08:44:49.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:21 2024 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:21 2024 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:21 2024 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: eb84190b-e551-45d5-8d58-e362047a6f7d 2024-08-31T08:44:49.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_37 snap 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:49.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.export 2024-08-31T08:44:49.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.export 2024-08-31T08:44:49.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.export 2024-08-31T08:44:49.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.export 2024-08-31T08:44:49.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:49.219 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.export 2024-08-31T08:44:49.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:49.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.export 2024-08-31T08:44:49.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_37.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_37.export 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_38 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_38 snap 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:49.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2024-08-31T08:44:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:50.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:50.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:51.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2024-08-31T08:44:51.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:51.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:51.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:51.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:51.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:51.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:51.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:51.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:51.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:51.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:51.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:51.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:51.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2024-08-31T08:44:51.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-08-31T08:44:51.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:51.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:51.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:51.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:44:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:44:51.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2024-08-31T08:44:52.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2024-08-31T08:44:52.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:52.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:52.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:44:52.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:52.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:44:52.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:44:52.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:52.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:52.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:44:52.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2024-08-31T08:44:52.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:44:52.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:44:52.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:44:52.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:44:52.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_38 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:52.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_38 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_38.status 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_38.status 2024-08-31T08:44:52.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.status 2024-08-31T08:44:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_38.status 2024-08-31T08:44:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_38.status 2024-08-31T08:44:52.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:52.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2024-08-31T08:44:52.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 160 2024-08-31T08:44:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.status 2024-08-31T08:44:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:52.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.status 2024-08-31T08:44:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2024-08-31T08:44:52.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:52.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:44:52.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=160 2024-08-31T08:44:52.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:52.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:52.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:44:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:53.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-08-31T08:44:53.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:53.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:53.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:53.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:53.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 160 = '' 2024-08-31T08:44:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:53.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:44:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:53.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-08-31T08:44:53.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:53.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:53.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 160 = '' 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:53.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-08-31T08:44:54.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:54.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:44:54.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:54.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 160 = '' 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:44:54.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:44:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:56.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.status 2024-08-31T08:44:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2024-08-31T08:44:56.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:44:56.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=160 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 160 = 160 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:56.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:44:56.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:57.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_38': 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11635e989fec 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11635e989fec 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:44:57.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:35 2024 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:35 2024 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:35 2024 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 94f6c5db-e9bd-4250-8d91-441f4a234f60 2024-08-31T08:44:57.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:44:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_38 snap 2024-08-31T08:44:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:44:57.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:44:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.export 2024-08-31T08:44:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.export 2024-08-31T08:44:57.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.export 2024-08-31T08:44:57.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.export 2024-08-31T08:44:57.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:57.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.export 2024-08-31T08:44:57.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:44:57.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.export 2024-08-31T08:44:57.672 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_38.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_38.export 2024-08-31T08:44:57.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_39 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_39 snap 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:44:57.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:44:58.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2024-08-31T08:44:58.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:58.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:58.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:44:59.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:44:59.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:44:59.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:59.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:59.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:44:59.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:44:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:44:59.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2024-08-31T08:44:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-08-31T08:44:59.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:44:59.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:44:59.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:59.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:44:59.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:44:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:44:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:44:59.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:00.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2024-08-31T08:45:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2024-08-31T08:45:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:00.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:00.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:00.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:00.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2024-08-31T08:45:00.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:00.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:00.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:00.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_39 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:00.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_39 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_39.status 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_39.status 2024-08-31T08:45:00.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.status 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_39.status 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_39.status 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:00.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2024-08-31T08:45:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:00.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2024-08-31T08:45:00.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 162 2024-08-31T08:45:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.status 2024-08-31T08:45:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:00.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.status 2024-08-31T08:45:00.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2024-08-31T08:45:00.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:00.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=162 2024-08-31T08:45:00.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:00.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:00.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:01.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:01.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:01.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:01.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:01.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:01.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-08-31T08:45:01.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:01.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:01.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:01.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 162 = '' 2024-08-31T08:45:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:01.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:01.593 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-08-31T08:45:01.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:01.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:01.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 162 = '' 2024-08-31T08:45:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:01.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-08-31T08:45:02.446 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:02.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:02.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 162 = '' 2024-08-31T08:45:02.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:02.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.status 2024-08-31T08:45:04.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2024-08-31T08:45:04.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:04.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=162 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 162 = 162 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:04.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:04.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:05.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_39': 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11632b0432d 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11632b0432d 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:05.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:45 2024 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:45 2024 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:45 2024 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6cf73648-3155-40fd-aa07-805332949efa 2024-08-31T08:45:05.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_39 snap 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.export 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.export 2024-08-31T08:45:05.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.export 2024-08-31T08:45:05.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.export 2024-08-31T08:45:05.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:05.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.export 2024-08-31T08:45:05.609 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:05.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.export 2024-08-31T08:45:05.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_39.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_39.export 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_40 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_40 snap 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:05.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:05.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:06.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:06.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:07.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2024-08-31T08:45:07.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:07.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:07.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:07.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:07.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:07.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:07.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:07.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:07.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:07.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:07.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:07.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:07.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:07.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:07.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:07.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2024-08-31T08:45:08.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-08-31T08:45:08.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:08.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:08.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:08.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:08.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:08.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:08.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:08.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2024-08-31T08:45:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2024-08-31T08:45:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:08.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2024-08-31T08:45:08.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:08.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:08.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:08.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:08.110 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:08.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:08.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:08.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:08.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:08.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:08.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2024-08-31T08:45:08.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:08.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:08.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_40 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:08.240 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_40 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:08.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_40.status 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_40.status 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.status 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_40.status 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_40.status 2024-08-31T08:45:08.242 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2024-08-31T08:45:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:08.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:08.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:08.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2024-08-31T08:45:08.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 164 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.status 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.status 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2024-08-31T08:45:08.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:08.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:08.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=164 2024-08-31T08:45:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:08.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:09.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:09.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:09.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:09.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:09.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:09.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-08-31T08:45:09.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:09.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:09.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:09.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:09.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:09.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:09.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:09.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 164 = '' 2024-08-31T08:45:09.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:09.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-08-31T08:45:09.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:09.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:09.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 164 = '' 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:09.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:10.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.status 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2024-08-31T08:45:10.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:10.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=164 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 164 = 164 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:10.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:10.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:11.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_40': 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163b0afb8b6 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163b0afb8b6 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:37:57 2024 2024-08-31T08:45:11.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:37:57 2024 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:37:57 2024 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: a0ff3644-b10e-4591-be88-e027ee2d4543 2024-08-31T08:45:11.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:11.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_40 snap 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.export 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.export 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.export 2024-08-31T08:45:11.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.export 2024-08-31T08:45:11.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:11.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.export 2024-08-31T08:45:12.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:12.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.export 2024-08-31T08:45:12.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_40.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_40.export 2024-08-31T08:45:12.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_41 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_41 snap 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:12.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:13.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:14.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:14.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:14.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:14.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:14.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2024-08-31T08:45:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:14.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:14.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:14.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:14.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:14.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:14.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:14.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:14.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:14.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2024-08-31T08:45:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-08-31T08:45:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:15.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2024-08-31T08:45:15.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:15.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:15.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:15.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:15.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:15.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:15.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:15.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2024-08-31T08:45:15.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2024-08-31T08:45:15.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:15.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:15.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:15.125 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:15.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:15.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:15.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:15.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2024-08-31T08:45:15.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:15.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:15.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:15.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:15.258 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_41 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:15.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_41 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_41.status 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_41.status 2024-08-31T08:45:15.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:15.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.status 2024-08-31T08:45:15.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_41.status 2024-08-31T08:45:15.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_41.status 2024-08-31T08:45:15.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:15.263 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2024-08-31T08:45:15.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 166 2024-08-31T08:45:15.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.status 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.status 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2024-08-31T08:45:15.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:15.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=166 2024-08-31T08:45:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:15.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-08-31T08:45:16.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:16.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:16.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:16.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:16.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:16.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 166 = '' 2024-08-31T08:45:16.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:16.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-08-31T08:45:16.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:16.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:16.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:16.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:16.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:16.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:16.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 166 = '' 2024-08-31T08:45:16.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:16.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-08-31T08:45:17.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:17.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:17.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:17.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 166 = '' 2024-08-31T08:45:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:17.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:45:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:19.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:19.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.status 2024-08-31T08:45:19.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2024-08-31T08:45:19.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:19.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=166 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 166 = 166 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:19.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:19.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:20.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2024-08-31T08:45:20.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_41': 2024-08-31T08:45:20.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:20.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163948eba49 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163948eba49 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:10 2024 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:10 2024 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:10 2024 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:20.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 8a05271f-7b34-4139-a110-04860d7d6a27 2024-08-31T08:45:20.269 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_41 snap 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:20.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:20.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.export 2024-08-31T08:45:20.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.export 2024-08-31T08:45:20.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.export 2024-08-31T08:45:20.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.export 2024-08-31T08:45:20.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:20.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.export 2024-08-31T08:45:20.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:20.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.export 2024-08-31T08:45:20.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_41.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_41.export 2024-08-31T08:45:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_42 2024-08-31T08:45:20.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_42 snap 2024-08-31T08:45:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:20.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:22.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:22.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:22.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:22.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:23.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:23.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:23.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:23.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:23.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:23.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:23.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:23.647 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2024-08-31T08:45:23.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-08-31T08:45:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:23.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:23.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:23.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:23.729 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:23.730 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:23.731 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:23.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:23.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:23.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:23.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2024-08-31T08:45:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2024-08-31T08:45:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:23.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:23.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_42 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:23.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:23.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:23.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:23.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:23.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:23.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:23.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_42 2024-08-31T08:45:23.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:23.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:23.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_42 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:23.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_42 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_42.status 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_42.status 2024-08-31T08:45:23.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.status 2024-08-31T08:45:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_42.status 2024-08-31T08:45:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_42.status 2024-08-31T08:45:23.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:23.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2024-08-31T08:45:24.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 168 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.status 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.status 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2024-08-31T08:45:24.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:24.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=168 2024-08-31T08:45:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:24.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-08-31T08:45:25.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:25.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:25.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:25.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 168 = '' 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:25.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:25.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-08-31T08:45:25.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:25.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:25.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:25.617 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 168 = '' 2024-08-31T08:45:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:25.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.status 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2024-08-31T08:45:26.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:26.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=168 2024-08-31T08:45:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 168 = 168 2024-08-31T08:45:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:26.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:26.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:27.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_42': 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163ee1d0bc2 2024-08-31T08:45:27.516 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163ee1d0bc2 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:24 2024 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:24 2024 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:24 2024 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 65a6d1ca-25ea-415c-b2bd-c2eaed4c4c4d 2024-08-31T08:45:27.517 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_42 snap 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.export 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.export 2024-08-31T08:45:27.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.export 2024-08-31T08:45:27.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.export 2024-08-31T08:45:27.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:27.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.export 2024-08-31T08:45:27.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:27.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.export 2024-08-31T08:45:28.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_42.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_42.export 2024-08-31T08:45:28.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_43 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_43 snap 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:28.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:29.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:29.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:29.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:29.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:30.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:30.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:30.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:30.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:30.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:30.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:30.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:30.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2024-08-31T08:45:30.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-08-31T08:45:30.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:30.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:30.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:30.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:30.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:31.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:31.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:31.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:31.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2024-08-31T08:45:31.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2024-08-31T08:45:31.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:31.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:31.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:31.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:31.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:31.102 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:31.103 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2024-08-31T08:45:31.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:31.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:31.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_43 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:31.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_43 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_43.status 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_43.status 2024-08-31T08:45:31.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.status 2024-08-31T08:45:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_43.status 2024-08-31T08:45:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_43.status 2024-08-31T08:45:31.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:31.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2024-08-31T08:45:31.279 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7452748653177698) > weight (0.25) 2024-08-31T08:45:31.279 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:45:31.279 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:45:31.279 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-08-31T08:45:31.280 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:45:31.279+0000 7f3b56327640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 43779) UID: 1000 2024-08-31T08:45:31.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 170 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.status 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.status 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2024-08-31T08:45:31.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:31.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=170 2024-08-31T08:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:31.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-08-31T08:45:32.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:32.183 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:32.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:32.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:32.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:32.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:32.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:32.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 170 = '' 2024-08-31T08:45:32.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:32.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:32.281 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:45:32.281 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:45:32.281 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:45:32.281 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:45:32.281 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:45:32.284 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:45:32.284 INFO:tasks.rbd_mirror_thrash:waiting for 61 secs before thrashing 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-08-31T08:45:32.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:32.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:32.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:32.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:32.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 170 = '' 2024-08-31T08:45:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:32.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:33.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.status 2024-08-31T08:45:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2024-08-31T08:45:33.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:33.484 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=170 2024-08-31T08:45:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 170 = 170 2024-08-31T08:45:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:33.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:33.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:34.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_43': 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163676647b6 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163676647b6 2024-08-31T08:45:34.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:34 2024 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:34 2024 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:34 2024 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1afe39a8-09d2-4199-bfe9-9908e5a224e5 2024-08-31T08:45:34.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_43 snap 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.export 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.export 2024-08-31T08:45:34.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.export 2024-08-31T08:45:34.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.export 2024-08-31T08:45:34.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:34.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.export 2024-08-31T08:45:34.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:34.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.export 2024-08-31T08:45:35.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_43.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_43.export 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_44 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_44 snap 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:35.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:36.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:36.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:36.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:37.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:37.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:37.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:37.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:37.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:37.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:37.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:37.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2024-08-31T08:45:38.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2024-08-31T08:45:38.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:38.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:38.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:38.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:38.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:38.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:38.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:38.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:38.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2024-08-31T08:45:38.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2024-08-31T08:45:38.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:38.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:38.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_44 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:38.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:38.100 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:38.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:38.117 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:38.118 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_44 2024-08-31T08:45:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:38.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:38.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:38.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:38.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_44 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:38.235 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_44 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_44.status 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_44.status 2024-08-31T08:45:38.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.status 2024-08-31T08:45:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_44.status 2024-08-31T08:45:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_44.status 2024-08-31T08:45:38.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:38.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2024-08-31T08:45:38.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 172 2024-08-31T08:45:38.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.status 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.status 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2024-08-31T08:45:38.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:38.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=172 2024-08-31T08:45:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:38.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:39.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-08-31T08:45:39.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:39.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:39.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:39.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:39.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:39.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:39.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:39.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 172 = '' 2024-08-31T08:45:39.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:39.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:39.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:39.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:39.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:39.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:39.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-08-31T08:45:39.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:39.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 172 = '' 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:39.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:40.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.status 2024-08-31T08:45:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2024-08-31T08:45:40.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:40.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:40.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=172 2024-08-31T08:45:40.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 172 = 172 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:40.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:41.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2024-08-31T08:45:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_44': 2024-08-31T08:45:41.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163c700d392 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163c700d392 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:41 2024 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:41 2024 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:41 2024 2024-08-31T08:45:41.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1d392638-d8a1-4ad5-a6fb-98b143b11843 2024-08-31T08:45:41.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:41.565 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_44 snap 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.export 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.export 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.export 2024-08-31T08:45:41.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.export 2024-08-31T08:45:41.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.export 2024-08-31T08:45:41.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:41.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.export 2024-08-31T08:45:42.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_44.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_44.export 2024-08-31T08:45:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_45 2024-08-31T08:45:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_45 snap 2024-08-31T08:45:42.079 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:42.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:42.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:42.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:42.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:43.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:43.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:44.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2024-08-31T08:45:44.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:44.936 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:44.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:44.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:44.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:44.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:44.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:44.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:44.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:44.961 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:44.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2024-08-31T08:45:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-08-31T08:45:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:45.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:45.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:45.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:45.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:45.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2024-08-31T08:45:45.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2024-08-31T08:45:45.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:45.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:45.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:45.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:45.141 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:45.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:45.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:45.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:45.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2024-08-31T08:45:45.270 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:45.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:45.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:45.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:45.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:45.273 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_45 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_45 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:45.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_45.status 2024-08-31T08:45:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_45.status 2024-08-31T08:45:45.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.status 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_45.status 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_45.status 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:45.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:45.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:45.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2024-08-31T08:45:45.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 174 2024-08-31T08:45:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.status 2024-08-31T08:45:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:45.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.status 2024-08-31T08:45:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2024-08-31T08:45:45.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:45.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:45.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=174 2024-08-31T08:45:45.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:45.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:45.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:46.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:46.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-08-31T08:45:46.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:46.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:46.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:46.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:46.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:46.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:46.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:46.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 174 = '' 2024-08-31T08:45:46.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:46.234 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:46.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:46.635 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-08-31T08:45:46.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:46.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:46.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 174 = '' 2024-08-31T08:45:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:46.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.status 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2024-08-31T08:45:47.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:47.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=174 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 174 = 174 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:47.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:47.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:47.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:47.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:47.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:48.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_45': 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116330efa44c 2024-08-31T08:45:48.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116330efa44c 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:49 2024 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:49 2024 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:49 2024 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 728a938d-32c1-4507-a5bd-b86a667a611c 2024-08-31T08:45:48.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_45 snap 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:48.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.export 2024-08-31T08:45:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.export 2024-08-31T08:45:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.export 2024-08-31T08:45:48.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.export 2024-08-31T08:45:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:48.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.export 2024-08-31T08:45:49.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:49.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.export 2024-08-31T08:45:49.281 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_45.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_45.export 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_46 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_46 snap 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:49.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:50.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:50.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:51.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2024-08-31T08:45:51.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:51.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:51.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:51.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:51.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:51.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:51.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2024-08-31T08:45:51.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:51.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:51.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:51.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:51.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:51.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:51.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:51.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:51.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:51.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:51.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2024-08-31T08:45:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-08-31T08:45:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:51.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:51.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:51.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:51.563 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:51.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:51.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:51.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2024-08-31T08:45:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2024-08-31T08:45:51.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:51.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:51.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:51.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:51.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:51.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:51.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2024-08-31T08:45:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:51.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:51.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_46 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:51.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_46 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_46.status 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_46.status 2024-08-31T08:45:51.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.status 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_46.status 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_46.status 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:51.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:51.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2024-08-31T08:45:52.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 176 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.status 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.status 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2024-08-31T08:45:52.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:52.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:45:52.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=176 2024-08-31T08:45:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:52.508 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-08-31T08:45:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:52.755 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:52.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=80 2024-08-31T08:45:52.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 176 = 80 2024-08-31T08:45:52.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:52.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:45:53.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:53.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:53.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:53.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:53.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:53.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-08-31T08:45:53.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:53.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:53.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 176 = '' 2024-08-31T08:45:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:53.207 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:54.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-08-31T08:45:54.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:54.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:45:54.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:54.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:45:54.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:45:54.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:45:54.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:45:54.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 176 = '' 2024-08-31T08:45:54.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:45:54.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:45:55.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.status 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2024-08-31T08:45:55.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:45:55.718 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=176 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 176 = 176 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:55.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:45:55.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:56.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2024-08-31T08:45:56.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_46': 2024-08-31T08:45:56.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:45:56.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 116311ec5064 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.116311ec5064 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:38:58 2024 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:38:58 2024 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:38:58 2024 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:45:56.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: dec3aa5e-1157-4af4-8203-e9297b75a332 2024-08-31T08:45:56.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:45:56.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:56.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_46 snap 2024-08-31T08:45:56.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.export 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.export 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.export 2024-08-31T08:45:56.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.export 2024-08-31T08:45:56.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:56.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.export 2024-08-31T08:45:57.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:45:57.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.export 2024-08-31T08:45:57.270 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_46.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_46.export 2024-08-31T08:45:57.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:57.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_47 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_47 snap 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:45:57.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:45:58.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2024-08-31T08:45:58.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:58.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:58.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:58.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:58.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:45:59.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2024-08-31T08:45:59.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:45:59.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:59.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:59.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:45:59.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:45:59.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:45:59.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:45:59.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:45:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:45:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:59.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:45:59.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:45:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:45:59.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2024-08-31T08:45:59.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2024-08-31T08:45:59.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:59.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2024-08-31T08:45:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:45:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:59.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:45:59.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:45:59.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:45:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:59.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:45:59.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2024-08-31T08:45:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2024-08-31T08:45:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:45:59.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:45:59.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:45:59.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:45:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:45:59.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:45:59.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:59.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:59.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:45:59.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2024-08-31T08:45:59.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:45:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:45:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:45:59.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:45:59.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:45:59.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_47 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_47 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:59.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:59.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_47.status 2024-08-31T08:45:59.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_47.status 2024-08-31T08:45:59.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:59.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.status 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_47.status 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_47.status 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:45:59.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:45:59.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:45:59.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2024-08-31T08:46:00.597 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55976c9c9c70 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977c8a0680 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977c8a0ea0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977e5d80d0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977e5d89c0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977f5a8a90 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9b52b0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d01a0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d0340 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x55977f9d1860 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x559795c5e9c0 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fc680 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fd040 2024-08-31T08:46:00.598 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba65b0 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba68f0 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba7a00 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba7ad0 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597b0dd5520 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597b0dd56c0 2024-08-31T08:46:00.599 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:46:00.595+0000 7fb313f7e640 -1 reset not still connected to 0x5597b0dd5ee0 2024-08-31T08:46:00.634 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 178 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.status 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.status 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2024-08-31T08:46:00.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:00.688 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:46:00.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=178 2024-08-31T08:46:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:00.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-08-31T08:46:00.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:00.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=81 2024-08-31T08:46:00.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 178 = 81 2024-08-31T08:46:00.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:00.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:01.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-08-31T08:46:01.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:01.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:01.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:01.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:01.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:01.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:01.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:01.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 178 = '' 2024-08-31T08:46:01.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:01.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:46:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:02.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:02.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:02.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:02.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-08-31T08:46:02.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:02.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:02.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 178 = '' 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:02.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:46:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:03.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.status 2024-08-31T08:46:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2024-08-31T08:46:03.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:03.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=178 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 178 = 178 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:46:03.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:04.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_47': 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 11637725d3c3 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.11637725d3c3 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:46:04.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:11 2024 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:11 2024 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:11 2024 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 47d42bc5-751d-4d5d-bba7-bc3264fb264b 2024-08-31T08:46:04.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_47 snap 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:46:04.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.export 2024-08-31T08:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.export 2024-08-31T08:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.export 2024-08-31T08:46:04.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.export 2024-08-31T08:46:05.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:05.163 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.export 2024-08-31T08:46:05.382 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:05.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.export 2024-08-31T08:46:05.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_47.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_47.export 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_48 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_48 snap 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:05.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:06.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:06.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2024-08-31T08:46:07.658 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:07.659 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:07.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:07.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:46:07.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:46:07.681 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:07.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:07.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:07.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2024-08-31T08:46:07.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-08-31T08:46:07.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:07.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:46:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:46:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:07.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:07.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2024-08-31T08:46:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2024-08-31T08:46:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:07.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:07.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:07.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:46:07.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:46:07.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:07.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:07.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:07.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2024-08-31T08:46:07.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:46:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:46:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:46:07.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:46:07.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:46:07.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_48 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_48 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:07.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_48.status 2024-08-31T08:46:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_48.status 2024-08-31T08:46:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:07.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.status 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_48.status 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_48.status 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:07.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2024-08-31T08:46:08.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 180 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.status 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.status 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2024-08-31T08:46:08.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:08.719 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:46:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=180 2024-08-31T08:46:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:08.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-08-31T08:46:08.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:08.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=82 2024-08-31T08:46:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 180 = 82 2024-08-31T08:46:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:08.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-08-31T08:46:09.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:09.422 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:09.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:09.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:09.423 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:09.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:09.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 180 = '' 2024-08-31T08:46:09.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:09.424 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:46:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:10.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:10.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-08-31T08:46:10.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:10.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:10.280 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:10.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:10.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:10.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 180 = '' 2024-08-31T08:46:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:10.283 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:46:11.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:11.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:11.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:11.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:11.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.status 2024-08-31T08:46:11.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2024-08-31T08:46:11.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:11.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:11.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=180 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 180 = 180 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:46:11.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:12.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_48': 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163ef8c93d5 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163ef8c93d5 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:46:12.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:24 2024 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:24 2024 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:24 2024 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6639998a-e2c5-467b-9e8b-80fb5fe10b2b 2024-08-31T08:46:12.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_48 snap 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.export 2024-08-31T08:46:12.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.export 2024-08-31T08:46:12.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.export 2024-08-31T08:46:12.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.export 2024-08-31T08:46:13.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:13.196 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.export 2024-08-31T08:46:13.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:13.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.export 2024-08-31T08:46:13.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_48.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_48.export 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_49 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_49 snap 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:13.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:46:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2024-08-31T08:46:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:14.684 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:14.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:15.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:46:15.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:46:15.687 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:46:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:46:15.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:15.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:15.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:46:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:46:15.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:46:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:15.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2024-08-31T08:46:15.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-08-31T08:46:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:15.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:15.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:15.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:46:15.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:46:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:15.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:15.814 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2024-08-31T08:46:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2024-08-31T08:46:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:15.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2024-08-31T08:46:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:46:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:15.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:46:15.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:15.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:46:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:46:15.894 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:46:15.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:15.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2024-08-31T08:46:16.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:46:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:46:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:46:16.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:46:16.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_49 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:46:16.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_49 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_49.status 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_49.status 2024-08-31T08:46:16.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.status 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_49.status 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_49.status 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2024-08-31T08:46:16.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:16.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:16.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:16.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2024-08-31T08:46:16.689 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 182 2024-08-31T08:46:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.status 2024-08-31T08:46:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:16.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.status 2024-08-31T08:46:16.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2024-08-31T08:46:16.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:16.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:46:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=182 2024-08-31T08:46:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:16.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-08-31T08:46:16.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:16.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:16.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=83 2024-08-31T08:46:16.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 182 = 83 2024-08-31T08:46:16.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:16.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:46:17.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:17.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:17.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:17.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:17.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:17.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-08-31T08:46:17.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:17.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:17.448 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:17.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 182 = '' 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:17.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:18.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-08-31T08:46:18.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:18.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:18.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:18.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:18.306 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:18.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:18.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 182 = '' 2024-08-31T08:46:18.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:18.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1.6 2024-08-31T08:46:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:19.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.status 2024-08-31T08:46:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2024-08-31T08:46:19.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:19.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=182 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 182 = 182 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:46:19.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:20.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_49': 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163ca9da606 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163ca9da606 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:46:21.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:33 2024 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:33 2024 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:33 2024 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: b080f365-a5bd-4cee-9717-8d5cc278ced7 2024-08-31T08:46:21.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_49 snap 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.export 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.export 2024-08-31T08:46:21.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.export 2024-08-31T08:46:21.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.export 2024-08-31T08:46:21.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:21.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.export 2024-08-31T08:46:21.479 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:21.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.export 2024-08-31T08:46:21.540 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_49.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_49.export 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_50 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_snap cluster2 mirror image_50 snap 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:21.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2024-08-31T08:46:22.373 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6741728961877913) > weight (0.25) 2024-08-31T08:46:22.373 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6805471366298077) > weight (0.25) 2024-08-31T08:46:22.373 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9652852645433755) > weight (0.25) 2024-08-31T08:46:22.373 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5736901815012295) > weight (0.25) 2024-08-31T08:46:22.373 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-08-31T08:46:22.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2024-08-31T08:46:22.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:22.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:22.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:22.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=started 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:22.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:23.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2024-08-31T08:46:23.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=started 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_result 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=stopped 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_instance=cluster1 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep -i state 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:46:23.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance=cluster1 2024-08-31T08:46:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local loop_instance 2024-08-31T08:46:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2024-08-31T08:46:23.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sleep 0 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' cluster1 '!=' cluster1 ']' 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ seq 0 3 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:0 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:0 cluster instance 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:0 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:46:23.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=0 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:0 cluster1 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:0 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:0 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:23.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=0 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 0 = cluster1:0 ']' 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=0 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=0 2024-08-31T08:46:23.714 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2024-08-31T08:46:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2024-08-31T08:46:23.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2024-08-31T08:46:23.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-08-31T08:46:23.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:23.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:23.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:23.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2024-08-31T08:46:23.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:1 2024-08-31T08:46:23.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:23.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:1 cluster instance 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:1 cluster1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:1 2024-08-31T08:46:23.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 1 = cluster1:1 ']' 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=1 2024-08-31T08:46:23.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2024-08-31T08:46:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:23.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:23.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2024-08-31T08:46:23.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2024-08-31T08:46:23.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2024-08-31T08:46:23.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd mirror status : get status for rbd mirror 2024-08-31T08:46:23.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:admin_socket: invalid command 2024-08-31T08:46:23.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2024-08-31T08:46:23.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2024-08-31T08:46:23.910 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1:2 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ shift 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set_cluster_instance cluster1:2 cluster instance 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local val=cluster1:2 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster_var_name=cluster 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local instance_var_name=instance 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=2 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval cluster=cluster1 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ eval instance=2 2024-08-31T08:46:23.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ daemon_asok_file cluster1:2 cluster1 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local local_cluster=cluster1:2 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster=cluster1 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local val=cluster1:2 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local cluster_var_name=local_cluster 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ local instance_var_name=instance 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ cluster=cluster1 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ instance=2 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ '[' 2 = cluster1:2 ']' 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval local_cluster=cluster1 2024-08-31T08:46:23.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ local_cluster=cluster1 2024-08-31T08:46:23.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ eval instance=2 2024-08-31T08:46:23.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ instance=2 2024-08-31T08:46:23.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2024-08-31T08:46:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2024-08-31T08:46:23.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2024-08-31T08:46:24.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ return 0 2024-08-31T08:46:24.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ status_result=' "state": "Replaying"' 2024-08-31T08:46:24.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo ' "state": "Replaying"' 2024-08-31T08:46:24.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ grep -i Replaying 2024-08-31T08:46:24.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "state": "Replaying" 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=started 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test started = started 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror image_50 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = journal ']' 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' snapshot = snapshot ']' 2024-08-31T08:46:24.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror image_50 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_cluster=cluster1 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster2-mirror-image_50.status 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster2-mirror-image_50.status 2024-08-31T08:46:24.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local status_log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.status 2024-08-31T08:46:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ mkfname cluster1-mirror-image_50.status 2024-08-31T08:46:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo cluster1-mirror-image_50.status 2024-08-31T08:46:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -e 's|[/ ]|_|g' 2024-08-31T08:46:24.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local local_status_log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2024-08-31T08:46:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:24.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2024-08-31T08:46:24.705 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Snapshot ID: 184 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.status 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.status 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2024-08-31T08:46:24.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:24.759 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/id 2024-08-31T08:46:24.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snapshot_id=184 2024-08-31T08:46:24.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:24.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:24.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.2 2024-08-31T08:46:24.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-08-31T08:46:24.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:25.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:25.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=84 2024-08-31T08:46:25.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 184 = 84 2024-08-31T08:46:25.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:25.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.4 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-08-31T08:46:25.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:46:25.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:-:1.1: Document is empty 2024-08-31T08:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: 2024-08-31T08:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:^ 2024-08-31T08:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id= 2024-08-31T08:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 184 = '' 2024-08-31T08:46:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2024-08-31T08:46:25.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.8 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local log=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.status 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2024-08-31T08:46:26.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2024-08-31T08:46:26.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local primary_snapshot_id=184 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 184 = 184 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2024-08-31T08:46:26.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 1 2024-08-31T08:46:27.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_50': 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 128 MiB in 32 objects 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 2 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1163210c39f3 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1163210c39f3 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:46:27.574 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:39:47 2024 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:39:47 2024 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:39:47 2024 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: protected: False 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1fd95124-6f6d-4ddf-a6ce-7b0aec68c31f 2024-08-31T08:46:27.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T08:46:27.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ compare_image_snaps mirror image_50 snap 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap_name=snap 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local ret=0 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local rmt_export=/tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.export 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local loc_export=/tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.export 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.export 2024-08-31T08:46:27.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.export 2024-08-31T08:46:27.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:27.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.export 2024-08-31T08:46:27.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2024-08-31T08:46:27.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cmp /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.export 2024-08-31T08:46:28.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rm -f /tmp/tmp.V6fpnqxu3N/cluster2-mirror-image_50.export /tmp/tmp.V6fpnqxu3N/cluster1-mirror-image_50.export 2024-08-31T08:46:28.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:28.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: delete many images' 2024-08-31T08:46:28.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:46:28.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:46:28.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:46:28 TEST: delete many images 2024-08-31T08:46:28.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:46:28 TEST: delete many images 2024-08-31T08:46:28.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 50 2024-08-31T08:46:28.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_1 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:28.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2024-08-31T08:46:28.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_1 2024-08-31T08:46:28.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:46:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:28.725 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_1 2024-08-31T08:46:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:46:28.726 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-08-31T08:46:28.769 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-08-31T08:46:28.885 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:46:28.884+0000 7f06d4b9a640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55843ac9ed00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:28.886 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:46:28.884+0000 7f06db3a7640 -1 rbd::mirror::ImageReplayer: 0x55843ba66280 [3/7317f9a4-8aa2-4900-b19f-fff053c553f4] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:30.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_2 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:30.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2024-08-31T08:46:31.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_2 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:31.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_2 2024-08-31T08:46:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:46:31.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-08-31T08:46:31.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-08-31T08:46:31.910 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:46:31.908+0000 7f06d539b640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55843b58cea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:31.911 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:46:31.908+0000 7f06dbba8640 -1 rbd::mirror::ImageReplayer: 0x55843b9cc500 [3/b9749456-99a0-4f9d-86ad-1654ff6a6a36] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:33.206 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.204+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c002d450 2024-08-31T08:46:33.206 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.204+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c3257ba0 2024-08-31T08:46:33.206 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.204+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c4d92ea0 2024-08-31T08:46:33.211 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.208+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c4d93ee0 2024-08-31T08:46:33.211 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.208+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c4e6c410 2024-08-31T08:46:33.212 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.210+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c5ca0dd0 2024-08-31T08:46:33.212 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.210+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c5ca0ea0 2024-08-31T08:46:33.212 INFO:tasks.ceph.cluster1.osd.0.smithi012.stderr:2024-08-31T08:46:33.210+0000 7f6c8966c640 -1 reset not still connected to 0x55b6c5ca15f0 2024-08-31T08:46:33.284 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-08-31T08:46:33.285 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-08-31T08:46:33.285 INFO:tasks.rbd_mirror_thrash:waiting for 4 secs before reviving daemons 2024-08-31T08:46:33.286 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:46:33.284+0000 7f06dc3a9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 38364) UID: 1000 2024-08-31T08:46:33.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_3 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:33.860 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2024-08-31T08:46:34.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_3 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:34.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_3 2024-08-31T08:46:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:46:34.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-08-31T08:46:34.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-08-31T08:46:35.032 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:35.030+0000 7f71fba62640 -1 rbd::mirror::ImageMap: 0x5624f3d58900 handle_peer_ack_remove: failed to remove global_image_id=c6165d76-3f8e-4fae-95c8-1c9ff3d6281d 2024-08-31T08:46:36.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_4 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:36.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2024-08-31T08:46:37.286 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-08-31T08:46:37.286 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:46:37.287 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-08-31T08:46:37.287 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-08-31T08:46:37.287 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-08-31T08:46:37.290 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-08-31T08:46:37.290 INFO:tasks.rbd_mirror_thrash:waiting for 113 secs before thrashing 2024-08-31T08:46:37.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_4 2024-08-31T08:46:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:46:37.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:37.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:37.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_4 2024-08-31T08:46:37.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:37.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:37.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:46:37.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-08-31T08:46:37.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-08-31T08:46:38.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:38.021+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4251d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:38.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:38.021+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7180 [3/6227f895-18e3-43af-ab10-bcfa84946b75] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:39.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:39.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:39.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_5 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:39.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2024-08-31T08:46:40.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:40.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_5 2024-08-31T08:46:40.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:46:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:40.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_5 2024-08-31T08:46:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:46:40.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-08-31T08:46:40.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-08-31T08:46:41.054 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:41.052+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3876820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:41.055 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:41.052+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4232000 [3/72388a70-106e-4eb6-a7f8-822bc6c40054] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:42.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:42.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_6 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2024-08-31T08:46:43.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_6 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:43.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_6 2024-08-31T08:46:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:46:43.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-08-31T08:46:43.911 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-08-31T08:46:44.024 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:44.023+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:44.024 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:44.023+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4233680 [3/9f6764b3-49ac-434c-9bd1-5079abbddbf0] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:44.571 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T08:46:44.569+0000 7f9dc3f75640 -1 reset not still connected to 0x55a0a6676f70 2024-08-31T08:46:46.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_7 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:46.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:46:46.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:46.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2024-08-31T08:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_7 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:46.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_7 2024-08-31T08:46:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:46:46.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-08-31T08:46:46.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-08-31T08:46:47.072 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:47.071+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3876680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:47.072 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:47.071+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4418c80 [3/be6ad6c7-3828-4fb6-ad59-75775c1277eb] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:48.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_8 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:48.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2024-08-31T08:46:49.895 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_8 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:49.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_8 2024-08-31T08:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:46:49.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-08-31T08:46:49.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-08-31T08:46:50.101 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:50.100+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3730000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:50.101 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:50.100+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4109900 [3/29308e73-15d8-45be-8eab-74c2808c5487] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:52.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_9 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:52.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2024-08-31T08:46:52.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_9 2024-08-31T08:46:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:52.926 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:46:52.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:52.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:52.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_9 2024-08-31T08:46:52.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:52.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:52.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:46:52.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-08-31T08:46:52.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-08-31T08:46:53.093 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:53.092+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f41984e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:53.093 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:53.092+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4233b80 [3/a832ab25-9838-4884-8321-006c701241dd] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:54.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_10 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:54.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2024-08-31T08:46:55.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_10 2024-08-31T08:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_10 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:55.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:46:55.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2024-08-31T08:46:55.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2024-08-31T08:46:56.106 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:56.105+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:56.106 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:56.105+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4108500 [3/138c0f6a-f5c2-4054-b5be-480015cc0759] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:46:58.048 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_11 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:58.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:46:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:46:58.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2024-08-31T08:46:58.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:46:58.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_11 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_11 2024-08-31T08:46:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:46:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:46:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:46:58.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2024-08-31T08:46:58.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2024-08-31T08:46:59.032 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:59.031+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402d380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:46:59.032 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:46:59.031+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6f00 [3/543fba89-43e8-4a84-a10b-ed65255deb74] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:01.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_12 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:01.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:01.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:47:01.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:01.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2024-08-31T08:47:01.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_12 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:01.967 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_12 2024-08-31T08:47:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:47:01.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2024-08-31T08:47:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2024-08-31T08:47:02.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:02.132+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402d520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:02.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:02.132+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4109b80 [3/32719a8b-c4e0-48d1-af14-d5d096fbbdaa] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:03.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_13 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:03.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:47:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:03.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2024-08-31T08:47:04.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:04.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_13 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_13 2024-08-31T08:47:04.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:47:04.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2024-08-31T08:47:05.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2024-08-31T08:47:05.153 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:05.152+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402cea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:05.153 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:05.152+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4233400 [3/942a444b-45e7-45c0-b6f0-0f1f234cac1e] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:07.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_14 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:47:07.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:07.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2024-08-31T08:47:07.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:07.974 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:07.973+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fd040 2024-08-31T08:47:07.974 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:07.973+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba68f0 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_14 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:07.997 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_14 2024-08-31T08:47:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:47:07.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2024-08-31T08:47:08.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2024-08-31T08:47:08.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:08.153+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:08.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:08.153+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4109400 [3/1ef17613-0e21-45d7-b5d2-ab9520155a3c] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:09.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_15 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:09.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2024-08-31T08:47:10.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_15 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:10.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_15 2024-08-31T08:47:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:47:10.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2024-08-31T08:47:11.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2024-08-31T08:47:11.145 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:11.144+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:11.145 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:11.144+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4192780 [3/ff694f40-7359-4371-9cac-7bf8444def68] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:13.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_16 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:13.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2024-08-31T08:47:13.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:13.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_16 2024-08-31T08:47:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:47:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:13.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_16 2024-08-31T08:47:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:47:13.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2024-08-31T08:47:13.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2024-08-31T08:47:14.028 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:14.027+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402c4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:14.028 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:14.027+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4108280 [3/09ffa3ae-165e-4e02-b27b-6c2e5c0d8dff] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:16.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:16.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_17 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:16.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2024-08-31T08:47:16.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_17 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:17.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:17.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_17 2024-08-31T08:47:17.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:17.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:17.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:47:17.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2024-08-31T08:47:17.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2024-08-31T08:47:17.163 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:17.162+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f402cb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:17.163 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:17.162+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4108c80 [3/188a5b44-125b-4376-a8fb-33c918538c19] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:18.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_18 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:18.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2024-08-31T08:47:19.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:19.997 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.995+0000 7fb313f7e640 -1 reset not still connected to 0x559795c5e9c0 2024-08-31T08:47:20.000 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fc680 2024-08-31T08:47:20.000 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597976fd040 2024-08-31T08:47:20.000 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba68f0 2024-08-31T08:47:20.000 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597a0ba7a00 2024-08-31T08:47:20.001 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597b0dd5520 2024-08-31T08:47:20.001 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:47:19.999+0000 7fb313f7e640 -1 reset not still connected to 0x5597b0dd56c0 2024-08-31T08:47:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_18 2024-08-31T08:47:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:20.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_18 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:20.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:47:20.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2024-08-31T08:47:20.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2024-08-31T08:47:20.195 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:20.194+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3730000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:20.195 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:20.194+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4233180 [3/9210fc52-db1b-4553-ad9c-5982d4188f4f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:22.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_19 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:47:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:22.113 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2024-08-31T08:47:23.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:23.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_19 2024-08-31T08:47:23.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:23.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:23.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:47:23.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:23.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:23.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_19 2024-08-31T08:47:23.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:23.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:23.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:47:23.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2024-08-31T08:47:23.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2024-08-31T08:47:23.191 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:23.190+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f41989c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:23.191 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:23.190+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419400 [3/d6dd38ac-296d-4d45-a399-c21cb2856fd7] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:24.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:24.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_20 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:24.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2024-08-31T08:47:25.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_20 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:25.871 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_20 2024-08-31T08:47:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:25.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:47:25.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2024-08-31T08:47:26.122 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2024-08-31T08:47:26.246 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:26.244+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f42516c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:26.247 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:26.244+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6000 [3/221ab01e-6802-4692-8a08-3ac552ff07a3] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:27.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_21 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:27.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2024-08-31T08:47:28.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:28.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_21 2024-08-31T08:47:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:47:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:28.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_21 2024-08-31T08:47:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:47:28.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2024-08-31T08:47:28.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2024-08-31T08:47:29.043 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:29.041+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:29.043 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:29.041+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6500 [3/83a21b96-117e-4747-bd14-d641899f4663] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:30.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_22 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:30.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:47:30.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:30.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2024-08-31T08:47:31.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_22 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:31.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:31.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_22 2024-08-31T08:47:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:47:31.917 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2024-08-31T08:47:31.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2024-08-31T08:47:32.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:32.077+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:32.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:32.077+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6c80 [3/500dccbe-6d7c-4195-84c9-4bdc2b07ec24] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:33.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:33.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_23 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2024-08-31T08:47:34.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:34.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_23 2024-08-31T08:47:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:47:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_23 2024-08-31T08:47:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:47:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2024-08-31T08:47:34.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2024-08-31T08:47:35.100 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:35.098+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4198ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:35.100 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:35.098+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4418500 [3/d80f7262-f0bd-4606-9216-6d6e2b02c77a] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:37.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_24 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:37.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2024-08-31T08:47:37.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_24 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:37.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_24 2024-08-31T08:47:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:47:37.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2024-08-31T08:47:37.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2024-08-31T08:47:38.106 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:38.105+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f42509c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:38.106 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:38.105+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4192500 [3/fca44610-a28e-42e0-b629-6dcbe524a929] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:39.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_25 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:39.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:39.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:47:39.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:39.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2024-08-31T08:47:40.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_25 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:40.956 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_25 2024-08-31T08:47:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:47:40.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2024-08-31T08:47:41.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2024-08-31T08:47:41.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:41.114+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4199040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:41.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:41.114+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4232c80 [3/8ee04712-4bc0-4a3d-a3bf-ac9627495068] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_26 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2024-08-31T08:47:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_26 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:43.883 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_26 2024-08-31T08:47:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:47:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2024-08-31T08:47:43.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2024-08-31T08:47:44.043 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:44.043+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:44.044 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:44.043+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4418000 [3/b5b3e638-f5aa-43cf-a2f1-98602c0046a0] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:46.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:46.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_27 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:46.067 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2024-08-31T08:47:46.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_27 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:46.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_27 2024-08-31T08:47:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:47:46.972 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2024-08-31T08:47:47.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2024-08-31T08:47:47.127 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:47.127+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:47.128 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:47.127+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6280 [3/23512681-0d1b-4424-93c6-c223988299d9] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:48.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_28 2024-08-31T08:47:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2024-08-31T08:47:48.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:47:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:48.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2024-08-31T08:47:49.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_28 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:49.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:49.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_28 2024-08-31T08:47:49.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:49.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:49.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:47:49.982 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2024-08-31T08:47:50.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2024-08-31T08:47:50.153 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:50.152+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f644e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:50.153 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:50.152+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6780 [3/32c899e0-aef6-4039-9156-ccc4a4e46708] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:52.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:52.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:52.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:52.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_29 2024-08-31T08:47:52.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2024-08-31T08:47:52.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:52.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:52.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:47:52.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:52.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2024-08-31T08:47:52.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:53.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_29 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:53.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_29 2024-08-31T08:47:53.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:53.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:53.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:47:53.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2024-08-31T08:47:53.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2024-08-31T08:47:53.156 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:53.155+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f42504e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:53.157 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:53.155+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4418280 [3/b9fca043-4527-4ec0-8fb0-13a1a765c247] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:54.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_30 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:47:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:54.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2024-08-31T08:47:55.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:55.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_30 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:55.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_30 2024-08-31T08:47:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:47:55.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2024-08-31T08:47:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2024-08-31T08:47:56.033 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:56.032+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f42501a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:56.033 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:56.032+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4192280 [3/f528088b-901a-428c-946e-f8201633ca46] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:47:57.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:47:57.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_31 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:47:57.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2024-08-31T08:47:58.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:47:58.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_31 2024-08-31T08:47:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:47:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:47:58.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:47:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_31 2024-08-31T08:47:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:47:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:47:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:47:58.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2024-08-31T08:47:58.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2024-08-31T08:47:59.044 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:59.043+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:47:59.044 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:47:59.043+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7900 [3/681647ab-f5ff-4fcf-8d6a-32f3fe7fa4ba] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:00.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:01.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_32 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:01.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2024-08-31T08:48:01.873 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:01.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_32 2024-08-31T08:48:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:48:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:01.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_32 2024-08-31T08:48:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:48:01.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2024-08-31T08:48:01.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2024-08-31T08:48:02.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:02.063+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4250b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:02.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:02.063+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4192000 [3/f34b2723-009a-42b7-ad38-540c04928627] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:03.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_33 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:03.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2024-08-31T08:48:04.374 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9950201464046518) > weight (0.25) 2024-08-31T08:48:04.374 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-08-31T08:48:04.375 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-08-31T08:48:04.375 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-08-31T08:48:04.376 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:48:04.375+0000 7fb463614640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 49276) UID: 1000 2024-08-31T08:48:04.882 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_33 2024-08-31T08:48:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:48:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:04.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:04.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_33 2024-08-31T08:48:04.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:04.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:04.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:48:04.907 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2024-08-31T08:48:04.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2024-08-31T08:48:05.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:05.062+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f4251a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:05.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:05.062+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4232780 [3/88c939cc-d595-4585-959a-55085fcd0473] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:05.376 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-08-31T08:48:05.376 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:48:05.377 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-08-31T08:48:05.377 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:48:05.377 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:48:05.380 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:48:05.380 INFO:tasks.rbd_mirror_thrash:waiting for 118 secs before thrashing 2024-08-31T08:48:07.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_34 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:48:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2024-08-31T08:48:07.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_34 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:07.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_34 2024-08-31T08:48:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:48:07.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2024-08-31T08:48:07.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2024-08-31T08:48:08.090 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:08.089+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f644e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:08.091 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:08.089+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419680 [3/d946a5bb-2842-4293-bff6-1ff598883e15] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:10.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_35 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:10.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2024-08-31T08:48:10.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_35 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:10.934 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_35 2024-08-31T08:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:48:10.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2024-08-31T08:48:10.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2024-08-31T08:48:11.095 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:11.094+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:11.095 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:11.094+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4419180 [3/c96b3a90-a801-484f-ba1a-ada1712c4cb4] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:13.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_36 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:13.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2024-08-31T08:48:13.862 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_36 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:13.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_36 2024-08-31T08:48:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:13.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:48:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2024-08-31T08:48:13.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2024-08-31T08:48:14.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:14.062+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5dd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:14.063 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:14.062+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4232500 [3/7c94795d-bc10-44e4-9ddb-b555399db9b4] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:16.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_37 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:16.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2024-08-31T08:48:16.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:16.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_37 2024-08-31T08:48:16.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:16.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:16.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:48:16.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_37 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:48:16.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2024-08-31T08:48:17.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2024-08-31T08:48:17.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:17.132+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f651e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:17.133 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:17.132+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419b80 [3/eb84190b-e551-45d5-8d58-e362047a6f7d] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:18.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_38 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:18.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:48:18.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:18.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2024-08-31T08:48:19.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_38 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:19.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_38 2024-08-31T08:48:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:19.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:48:19.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2024-08-31T08:48:20.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2024-08-31T08:48:20.139 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:20.138+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f65040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:20.140 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:20.138+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4232f00 [3/94f6c5db-e9bd-4250-8d91-441f4a234f60] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:22.082 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_39 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:22.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:48:22.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:22.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2024-08-31T08:48:22.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:22.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_39 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_39 2024-08-31T08:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2024-08-31T08:48:23.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2024-08-31T08:48:23.158 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:23.156+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:23.158 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:23.156+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7b80 [3/6cf73648-3155-40fd-aa07-805332949efa] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:24.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:24.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_40 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:24.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2024-08-31T08:48:25.861 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_40 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:25.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_40 2024-08-31T08:48:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:48:25.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2024-08-31T08:48:26.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2024-08-31T08:48:26.252 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:26.250+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:26.253 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:26.250+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4233900 [3/a0ff3644-b10e-4591-be88-e027ee2d4543] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:28.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_41 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:28.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2024-08-31T08:48:28.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_41 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:28.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:28.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_41 2024-08-31T08:48:28.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:28.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:28.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:48:28.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2024-08-31T08:48:28.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2024-08-31T08:48:29.052 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:29.051+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a94b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:29.052 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:29.051+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4232a00 [3/8a05271f-7b34-4139-a110-04860d7d6a27] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:30.291 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2024-08-31T08:48:30.292 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2024-08-31T08:48:30.292 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:48:30.293 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.smithi167.stderr:2024-08-31T08:48:30.292+0000 7f2e83150640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 53048) UID: 1000 2024-08-31T08:48:31.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_42 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:48:31.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:31.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2024-08-31T08:48:31.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:31.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_42 2024-08-31T08:48:31.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:48:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:31.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_42 2024-08-31T08:48:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:48:31.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2024-08-31T08:48:31.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2024-08-31T08:48:32.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:32.091+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a949c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:32.092 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:32.091+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7400 [3/65a6d1ca-25ea-415c-b2bd-c2eaed4c4c4d] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:33.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:33.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_43 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:33.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2024-08-31T08:48:34.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:34.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_43 2024-08-31T08:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_43 2024-08-31T08:48:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:48:34.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2024-08-31T08:48:34.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2024-08-31T08:48:35.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:35.095+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:35.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:35.095+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4108f00 [3/1afe39a8-09d2-4199-bfe9-9908e5a224e5] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:35.293 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2024-08-31T08:48:35.293 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:48:35.294 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2024-08-31T08:48:35.294 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2024-08-31T08:48:35.294 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2024-08-31T08:48:35.297 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2024-08-31T08:48:35.297 INFO:tasks.rbd_mirror_thrash:waiting for 103 secs before thrashing 2024-08-31T08:48:37.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_44 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:48:37.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:37.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2024-08-31T08:48:37.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_44 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:37.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_44 2024-08-31T08:48:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:48:37.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2024-08-31T08:48:38.188 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2024-08-31T08:48:38.303 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:38.302+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f62680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:38.303 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:38.302+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4109180 [3/1d392638-d8a1-4ad5-a6fb-98b143b11843] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:39.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_45 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:48:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2024-08-31T08:48:40.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_45 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:40.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_45 2024-08-31T08:48:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:48:40.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2024-08-31T08:48:40.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2024-08-31T08:48:41.108 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:41.107+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5e000 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:41.108 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:41.107+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4232280 [3/728a938d-32c1-4507-a5bd-b86a667a611c] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:43.052 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_46 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:43.053 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2024-08-31T08:48:43.874 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_46 2024-08-31T08:48:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:48:43.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_46 2024-08-31T08:48:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:43.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:43.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:48:43.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2024-08-31T08:48:43.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2024-08-31T08:48:44.053 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:44.052+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3735380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:44.053 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:44.052+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/dec3aa5e-1157-4af4-8203-e9297b75a332] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:46.065 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_47 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:46.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2024-08-31T08:48:46.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_47 2024-08-31T08:48:46.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_47 2024-08-31T08:48:46.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:46.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:46.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:48:46.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2024-08-31T08:48:47.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2024-08-31T08:48:47.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:47.130+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f37351e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:47.131 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:47.130+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e6a00 [3/47d42bc5-751d-4d5d-bba7-bc3264fb264b] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:48.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_48 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:48.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2024-08-31T08:48:49.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_48 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:49.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_48 2024-08-31T08:48:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:48:49.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2024-08-31T08:48:50.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2024-08-31T08:48:50.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:50.153+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3735040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:50.154 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:50.153+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/6639998a-e2c5-467b-9e8b-80fb5fe10b2b] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:52.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_49 2024-08-31T08:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2024-08-31T08:48:52.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:48:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:52.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2024-08-31T08:48:52.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_49 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:52.998 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:52.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_49 2024-08-31T08:48:52.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:52.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:52.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:48:52.999 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2024-08-31T08:48:53.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2024-08-31T08:48:53.160 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:53.158+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3734b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:53.160 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:53.158+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/b080f365-a5bd-4cee-9717-8d5cc278ced7] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:54.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_50 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local snap=snap 2024-08-31T08:48:54.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2024-08-31T08:48:55.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing snap: 100% complete...done. 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_50 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:48:55.892 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:48:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_50 2024-08-31T08:48:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:48:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:48:55.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2024-08-31T08:48:55.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2024-08-31T08:48:56.047 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:56.046+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3877520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:48:56.047 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:48:56.046+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4109680 [3/1fd95124-6f6d-4ddf-a6ce-7b0aec68c31f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:48:58.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-08-31T08:48:58.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: image deletions should propagate' 2024-08-31T08:48:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:48:58.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:48:58.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:48:58 TEST: image deletions should propagate 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:48:58 TEST: image deletions should propagate 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_images cluster1 mirror 0 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_count=0 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local count 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local last_count=0 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ true 2024-08-31T08:48:58.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-08-31T08:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'images: ' 2024-08-31T08:48:58.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 2 2024-08-31T08:48:58.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:48:58.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ count=0 2024-08-31T08:48:58.044 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 0 = 0 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state 2024-08-31T08:48:58.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 40 2024-08-31T08:48:58.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in `seq 1 40` 2024-08-31T08:48:58.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test 1 -ne 1 2024-08-31T08:48:58.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2024-08-31T08:48:58.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ grep 'image health:' 2024-08-31T08:48:58.046 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cut '-d ' -f 3 2024-08-31T08:48:58.081 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: failed to query services: (13) Permission denied 2024-08-31T08:48:58.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ state=OK 2024-08-31T08:48:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = ERROR 2024-08-31T08:48:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test OK = OK 2024-08-31T08:48:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 50 2024-08-31T08:48:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.085 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:48:58.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:48:58.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.118 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-08-31T08:48:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:58.121 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:58.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:48:58.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:48:58.225 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-08-31T08:48:58.260 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_2 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:48:58.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-08-31T08:48:58.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:58.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:58.298 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:58.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:48:58.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.435 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-08-31T08:48:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_3 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:48:58.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-08-31T08:48:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:58.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-08-31T08:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:48:58.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:48:58.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:48:58.579 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.610 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-08-31T08:48:58.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:58.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_4 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:48:58.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:48:58.616 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-08-31T08:48:58.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:58.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:58.651 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-08-31T08:48:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:48:58.754 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_5 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:48:58.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:48:58.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-08-31T08:48:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:58.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-08-31T08:48:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:48:58.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:48:58.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:58.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-08-31T08:48:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:58.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_6 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:48:58.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:48:58.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.001 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-08-31T08:48:59.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.004 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:48:59.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_7 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:48:59.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:48:59.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-08-31T08:48:59.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-08-31T08:48:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:48:59.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_8 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:48:59.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:48:59.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-08-31T08:48:59.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.466 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:48:59.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.498 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-08-31T08:48:59.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:59.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:59.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_9 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.503 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:48:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:48:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:48:59.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-08-31T08:48:59.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-08-31T08:48:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:48:59.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.674 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:59.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_10 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_10 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_10 2024-08-31T08:48:59.679 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-08-31T08:48:59.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-08-31T08:48:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_10 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_10 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-08-31T08:48:59.818 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_10: (2) No such file or directory 2024-08-31T08:48:59.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:48:59.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_11 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_11 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_11 2024-08-31T08:48:59.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-08-31T08:48:59.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:48:59.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-08-31T08:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:48:59.893 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_11 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:48:59.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_11 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_11 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2024-08-31T08:48:59.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_11: (2) No such file or directory 2024-08-31T08:49:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.034 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_12 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_12 2024-08-31T08:49:00.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_12 2024-08-31T08:49:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-08-31T08:49:00.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.066 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-08-31T08:49:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:00.171 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_12 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_12 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_12 2024-08-31T08:49:00.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2024-08-31T08:49:00.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_12: (2) No such file or directory 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_13 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:49:00.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_13 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_13 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-08-31T08:49:00.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-08-31T08:49:00.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_13 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:00.358 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_13 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_13 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2024-08-31T08:49:00.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_13: (2) No such file or directory 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_14 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:49:00.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_14 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_14 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-08-31T08:49:00.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-08-31T08:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:00.533 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_14 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_14 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_14 2024-08-31T08:49:00.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2024-08-31T08:49:00.535 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.568 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_14: (2) No such file or directory 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_15 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.572 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_15 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_15 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-08-31T08:49:00.573 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.605 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-08-31T08:49:00.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_15 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:00.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_15 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_15 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2024-08-31T08:49:00.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.743 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_15: (2) No such file or directory 2024-08-31T08:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_16 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_16 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_16 2024-08-31T08:49:00.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-08-31T08:49:00.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-08-31T08:49:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_16 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_16 2024-08-31T08:49:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_16 2024-08-31T08:49:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2024-08-31T08:49:00.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_16: (2) No such file or directory 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:00.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_17 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_17 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_17 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-08-31T08:49:00.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:00.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-08-31T08:49:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:00.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2024-08-31T08:49:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_17 2024-08-31T08:49:01.060 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_17 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_17 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2024-08-31T08:49:01.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.295 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_17: (2) No such file or directory 2024-08-31T08:49:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:01.299 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_18 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.300 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_18 2024-08-31T08:49:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_18 2024-08-31T08:49:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-08-31T08:49:01.301 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.333 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-08-31T08:49:01.335 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:01.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_18 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_18 2024-08-31T08:49:01.438 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_18 2024-08-31T08:49:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2024-08-31T08:49:01.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_18: (2) No such file or directory 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_19 2024-08-31T08:49:01.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_19 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_19 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-08-31T08:49:01.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.509 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-08-31T08:49:01.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:01.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:01.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2024-08-31T08:49:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_19 2024-08-31T08:49:01.614 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_19 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_19 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2024-08-31T08:49:01.615 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_19: (2) No such file or directory 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_20 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.655 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_20 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_20 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-08-31T08:49:01.656 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-08-31T08:49:01.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:01.694 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2024-08-31T08:49:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_20 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_20 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_20 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2024-08-31T08:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_20: (2) No such file or directory 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_21 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.833 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_21 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_21 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-08-31T08:49:01.834 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:01.865 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-08-31T08:49:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:01.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_21 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:01.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_21 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_21 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2024-08-31T08:49:01.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_21: (2) No such file or directory 2024-08-31T08:49:02.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:02.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:02.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_22 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_22 2024-08-31T08:49:02.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_22 2024-08-31T08:49:02.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-08-31T08:49:02.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.045 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-08-31T08:49:02.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:02.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:02.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:02.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_22 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_22 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.151 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_22 2024-08-31T08:49:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2024-08-31T08:49:02.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_22: (2) No such file or directory 2024-08-31T08:49:02.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_23 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_23 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_23 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-08-31T08:49:02.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-08-31T08:49:02.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:02.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:02.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2024-08-31T08:49:02.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_23 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_23 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_23 2024-08-31T08:49:02.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2024-08-31T08:49:02.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_23: (2) No such file or directory 2024-08-31T08:49:02.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:02.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:02.366 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_24 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:02.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_24 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_24 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-08-31T08:49:02.368 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-08-31T08:49:02.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:02.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:02.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2024-08-31T08:49:02.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_24 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_24 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.506 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_24 2024-08-31T08:49:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2024-08-31T08:49:02.507 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.539 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_24: (2) No such file or directory 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:02.543 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_25 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_25 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_25 2024-08-31T08:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-08-31T08:49:02.545 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-08-31T08:49:02.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:02.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:02.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2024-08-31T08:49:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.682 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_25 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_25 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_25 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2024-08-31T08:49:02.683 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.716 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_25: (2) No such file or directory 2024-08-31T08:49:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:02.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_26 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_26 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_26 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-08-31T08:49:02.722 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:02.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-08-31T08:49:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:02.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_26 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_26 2024-08-31T08:49:02.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_26 2024-08-31T08:49:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2024-08-31T08:49:02.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_26: (2) No such file or directory 2024-08-31T08:49:03.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:03.097 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_27 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.098 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_27 2024-08-31T08:49:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_27 2024-08-31T08:49:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-08-31T08:49:03.099 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-08-31T08:49:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:03.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:03.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2024-08-31T08:49:03.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_27 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_27 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_27 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2024-08-31T08:49:03.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_27: (2) No such file or directory 2024-08-31T08:49:03.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_28 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_28 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_28 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-08-31T08:49:03.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.310 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-08-31T08:49:03.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:03.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:03.311 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2024-08-31T08:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_28 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_28 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_28 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2024-08-31T08:49:03.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.449 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_28: (2) No such file or directory 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_29 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_29 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_29 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-08-31T08:49:03.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.486 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-08-31T08:49:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:03.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2024-08-31T08:49:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_29 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_29 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_29 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2024-08-31T08:49:03.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_29: (2) No such file or directory 2024-08-31T08:49:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:03.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_30 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_30 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_30 2024-08-31T08:49:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-08-31T08:49:03.630 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:03.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-08-31T08:49:03.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:03.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:03.665 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2024-08-31T08:49:03.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_30 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_30 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_30 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2024-08-31T08:49:03.767 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_30: (2) No such file or directory 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_31 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_31 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_31 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-08-31T08:49:04.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-08-31T08:49:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:04.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2024-08-31T08:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_31 2024-08-31T08:49:04.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_31 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_31 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2024-08-31T08:49:04.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_31: (2) No such file or directory 2024-08-31T08:49:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_32 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_32 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_32 2024-08-31T08:49:04.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-08-31T08:49:04.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-08-31T08:49:04.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:04.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:04.221 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2024-08-31T08:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_32 2024-08-31T08:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_32 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_32 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2024-08-31T08:49:04.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.357 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_32: (2) No such file or directory 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_33 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2024-08-31T08:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_33 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_33 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-08-31T08:49:04.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.394 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-08-31T08:49:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:04.397 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2024-08-31T08:49:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_33 2024-08-31T08:49:04.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_33 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_33 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2024-08-31T08:49:04.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_33: (2) No such file or directory 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_34 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:04.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_34 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_34 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-08-31T08:49:04.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-08-31T08:49:04.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:04.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:04.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:04.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2024-08-31T08:49:04.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_34 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_34 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_34 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2024-08-31T08:49:04.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_34: (2) No such file or directory 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_35 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:04.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_35 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_35 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-08-31T08:49:04.919 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:04.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-08-31T08:49:04.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:04.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:04.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2024-08-31T08:49:05.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_35 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_35 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_35 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2024-08-31T08:49:05.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.101 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_35: (2) No such file or directory 2024-08-31T08:49:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_36 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_36 2024-08-31T08:49:05.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_36 2024-08-31T08:49:05.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-08-31T08:49:05.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-08-31T08:49:05.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:05.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:05.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_36 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_36 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_36 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2024-08-31T08:49:05.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.282 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_36: (2) No such file or directory 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_37 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_37 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_37 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-08-31T08:49:05.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.320 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-08-31T08:49:05.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:05.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:05.323 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_37 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_37 2024-08-31T08:49:05.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_37 2024-08-31T08:49:05.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2024-08-31T08:49:05.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_37: (2) No such file or directory 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_38 2024-08-31T08:49:05.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_38 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_38 2024-08-31T08:49:05.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-08-31T08:49:05.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.497 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-08-31T08:49:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:05.500 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_38 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_38 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_38 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2024-08-31T08:49:05.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.636 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_38: (2) No such file or directory 2024-08-31T08:49:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_39 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:49:05.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_39 2024-08-31T08:49:05.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_39 2024-08-31T08:49:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-08-31T08:49:05.645 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-08-31T08:49:05.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:05.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:05.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_39 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_39 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_39 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2024-08-31T08:49:05.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_39: (2) No such file or directory 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_40 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_40 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_40 2024-08-31T08:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-08-31T08:49:05.817 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-08-31T08:49:05.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:05.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:05.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_40 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:05.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_40 2024-08-31T08:49:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_40 2024-08-31T08:49:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2024-08-31T08:49:05.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:05.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_40: (2) No such file or directory 2024-08-31T08:49:05.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:05.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_41 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:05.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_41 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_41 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-08-31T08:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-08-31T08:49:06.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:06.129 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_41 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:06.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_41 2024-08-31T08:49:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_41 2024-08-31T08:49:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2024-08-31T08:49:06.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_41: (2) No such file or directory 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_42 2024-08-31T08:49:06.168 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_42 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_42 2024-08-31T08:49:06.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-08-31T08:49:06.170 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-08-31T08:49:06.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:06.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2024-08-31T08:49:06.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_42 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_42 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_42 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2024-08-31T08:49:06.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.343 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_42: (2) No such file or directory 2024-08-31T08:49:06.347 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_43 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_43 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_43 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-08-31T08:49:06.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.381 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-08-31T08:49:06.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.385 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_43 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_43 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_43 2024-08-31T08:49:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2024-08-31T08:49:06.488 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_43: (2) No such file or directory 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_44 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_44 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_44 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-08-31T08:49:06.526 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-08-31T08:49:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.561 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2024-08-31T08:49:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_44 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_44 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_44 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2024-08-31T08:49:06.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_44: (2) No such file or directory 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_45 2024-08-31T08:49:06.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_45 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_45 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-08-31T08:49:06.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-08-31T08:49:06.739 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.740 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_45 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_45 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_45 2024-08-31T08:49:06.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2024-08-31T08:49:06.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_45: (2) No such file or directory 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_46 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2024-08-31T08:49:06.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_46 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:06.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_46 2024-08-31T08:49:06.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-08-31T08:49:06.881 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:06.912 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-08-31T08:49:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:06.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:07.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_46 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_46 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_46 2024-08-31T08:49:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2024-08-31T08:49:07.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_46: (2) No such file or directory 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_47 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_47 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_47 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-08-31T08:49:07.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-08-31T08:49:07.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:07.092 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:07.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:07.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2024-08-31T08:49:07.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_47 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_47 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_47 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2024-08-31T08:49:07.195 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_47: (2) No such file or directory 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:07.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_48 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_48 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_48 2024-08-31T08:49:07.232 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-08-31T08:49:07.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.265 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-08-31T08:49:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:07.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:07.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2024-08-31T08:49:07.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_48 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_48 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_48 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2024-08-31T08:49:07.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_48: (2) No such file or directory 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_49 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_49 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_49 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-08-31T08:49:07.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-08-31T08:49:07.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:07.444 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:07.445 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:07.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_49 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_49 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_49 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2024-08-31T08:49:07.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.581 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_49: (2) No such file or directory 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_50 2024-08-31T08:49:07.585 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_50 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_50 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-08-31T08:49:07.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.618 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-08-31T08:49:07.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:07.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:07.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_50 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:07.723 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_50 2024-08-31T08:49:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_50 2024-08-31T08:49:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2024-08-31T08:49:07.724 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:07.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_50: (2) No such file or directory 2024-08-31T08:49:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:07.760 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ testlog 'TEST: delete images during bootstrap' 2024-08-31T08:49:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ tee -a /tmp/tmp.V6fpnqxu3N/rbd-mirror.test.log 2024-08-31T08:49:07.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ date '+%F %T' 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ echo 2024-08-31 08:49:07 TEST: delete images during bootstrap 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31 08:49:07 TEST: delete images during bootstrap 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_pool_mirror_mode cluster1 mirror image 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=image 2024-08-31T08:49:07.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2024-08-31T08:49:07.795 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd: mirroring is already configured for image mode 2024-08-31T08:49:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_pool_mirror_mode cluster2 mirror image 2024-08-31T08:49:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=image 2024-08-31T08:49:07.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2024-08-31T08:49:07.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd: mirroring is already configured for image mode 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ start_mirror cluster1 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set_cluster_instance cluster1 cluster instance 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local val=cluster1 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster_var_name=cluster 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local instance_var_name=instance 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ cluster=cluster1 2024-08-31T08:49:07.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=cluster1 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' cluster1 = cluster1 ']' 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ instance=0 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval cluster=cluster1 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cluster=cluster1 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ eval instance=0 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ instance=0 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=test 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ seq 1 10 2024-08-31T08:49:07.836 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_1 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_1 512M 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:07.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_1 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_1 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:49:07.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2024-08-31T08:49:08.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:49:08.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_1': 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 189d2af79606 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.189d2af79606 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:49:07 2024 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:49:07 2024 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:49:07 2024 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 4dc58479-be23-4acf-8087-3091c1bbb4e9 2024-08-31T08:49:08.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:49:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_1 2024-08-31T08:49:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:08.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:49:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=11 2024-08-31T08:49:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:49:08.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:49:09.218 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 27814912 bytes 2024-08-31T08:49:09.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 91761664 bytes 2024-08-31T08:49:09.623 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 143609856 bytes 2024-08-31T08:49:09.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 184366080 bytes 2024-08-31T08:49:09.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 203487232 bytes 2024-08-31T08:49:10.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 220783616 bytes 2024-08-31T08:49:10.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 233900032 bytes 2024-08-31T08:49:10.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 243476480 bytes 2024-08-31T08:49:10.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 260103168 bytes 2024-08-31T08:49:10.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 330691584 bytes 2024-08-31T08:49:10.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 356995072 bytes 2024-08-31T08:49:10.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 417199104 bytes 2024-08-31T08:49:11.076 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 456198144 bytes 2024-08-31T08:49:11.184 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 468688896 bytes 2024-08-31T08:49:11.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 485628928 bytes 2024-08-31T08:49:11.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 563269632 bytes 2024-08-31T08:49:11.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 633029632 bytes 2024-08-31T08:49:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 712965120 bytes 2024-08-31T08:49:12.148 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 755396608 bytes 2024-08-31T08:49:12.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 794101760 bytes 2024-08-31T08:49:12.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 825376768 bytes 2024-08-31T08:49:12.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 865774592 bytes 2024-08-31T08:49:12.832 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 907736064 bytes 2024-08-31T08:49:12.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 936488960 bytes 2024-08-31T08:49:13.064 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 937133056 bytes 2024-08-31T08:49:13.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 972907520 bytes 2024-08-31T08:49:13.433 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1033736192 bytes 2024-08-31T08:49:13.620 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1068371968 bytes 2024-08-31T08:49:13.715 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1088858112 bytes 2024-08-31T08:49:13.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1125264384 bytes 2024-08-31T08:49:14.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1161532416 bytes 2024-08-31T08:49:14.262 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1208224768 bytes 2024-08-31T08:49:14.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1301253120 bytes 2024-08-31T08:49:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1371149312 bytes 2024-08-31T08:49:14.922 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1403501568 bytes 2024-08-31T08:49:15.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1425216512 bytes 2024-08-31T08:49:15.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1499727872 bytes 2024-08-31T08:49:15.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1524844544 bytes 2024-08-31T08:49:15.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1589874688 bytes 2024-08-31T08:49:15.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1645200384 bytes 2024-08-31T08:49:16.084 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1708181504 bytes 2024-08-31T08:49:16.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1768991744 bytes 2024-08-31T08:49:16.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1809556480 bytes 2024-08-31T08:49:16.644 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1853559808 bytes 2024-08-31T08:49:16.868 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1905124352 bytes 2024-08-31T08:49:17.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1973040128 bytes 2024-08-31T08:49:17.302 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2024290304 bytes 2024-08-31T08:49:17.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2089428992 bytes 2024-08-31T08:49:17.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2110032896 bytes 2024-08-31T08:49:17.878 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2182115328 bytes 2024-08-31T08:49:17.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2182721536 bytes 2024-08-31T08:49:18.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2289801216 bytes 2024-08-31T08:49:18.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2369858560 bytes 2024-08-31T08:49:18.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2436350976 bytes 2024-08-31T08:49:19.006 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2463537152 bytes 2024-08-31T08:49:19.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2498470912 bytes 2024-08-31T08:49:19.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2576803840 bytes 2024-08-31T08:49:19.633 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2619249664 bytes 2024-08-31T08:49:19.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2646267904 bytes 2024-08-31T08:49:19.915 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2664647680 bytes 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:19.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:49:19.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:20.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=1124233095b9 2024-08-31T08:49:20.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:20.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_1 present 1124233095b9 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:49:20.105 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=1124233095b9 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:49:20.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=1124233095b9 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 1124233095b9 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 1124233095b9 -o 1124233095b9 = 1124233095b9 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_1 2024-08-31T08:49:20.149 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2024-08-31T08:49:22.405 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:49:22.404+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3732680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:49:22.405 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:49:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/4dc58479-be23-4acf-8087-3091c1bbb4e9] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:49:51.689 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:49:51.687+0000 7f9b92915640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x559c363f0800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=185 2024-08-31T08:49:51.690 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:49:51.687+0000 7f9b92915640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x559c363f0800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=186, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=b9, last_copied_object_number=128, snap_seqs={185=18446744073709551614}] 2024-08-31T08:49:51.704 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:49:51.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2024-08-31T08:49:51.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:49:51.721 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:51.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-08-31T08:49:51.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:49:51.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:51.761 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:51.863 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_1 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_1 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2024-08-31T08:49:51.864 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:51.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_1: (2) No such file or directory 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_1 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:51.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_1 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:49:51.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:49:51.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_1 2024-08-31T08:49:51.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:51.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:51.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_1 2024-08-31T08:49:51.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2024-08-31T08:49:51.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2024-08-31T08:49:52.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_2 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_2 512M 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:52.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_2 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_2 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:49:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2024-08-31T08:49:52.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:49:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_2': 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 18b1d8460674 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.18b1d8460674 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:49:52.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:49:52 2024 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:49:52 2024 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:49:52 2024 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc 2024-08-31T08:49:52.748 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:49:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_2 2024-08-31T08:49:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:52.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:49:52.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:49:52.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:49:52.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:49:53.026 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 67173376 bytes 2024-08-31T08:49:53.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 88708096 bytes 2024-08-31T08:49:53.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 131269632 bytes 2024-08-31T08:49:53.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 167942144 bytes 2024-08-31T08:49:53.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 267172864 bytes 2024-08-31T08:49:53.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 299338752 bytes 2024-08-31T08:49:54.047 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 307792896 bytes 2024-08-31T08:49:54.139 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 313822208 bytes 2024-08-31T08:49:54.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 349193216 bytes 2024-08-31T08:49:54.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 410343424 bytes 2024-08-31T08:49:54.654 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 439837696 bytes 2024-08-31T08:49:54.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 470408192 bytes 2024-08-31T08:49:54.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 482270208 bytes 2024-08-31T08:49:55.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 518777856 bytes 2024-08-31T08:49:55.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 589449216 bytes 2024-08-31T08:49:55.361 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 592585728 bytes 2024-08-31T08:49:55.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 632514560 bytes 2024-08-31T08:49:55.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 666638336 bytes 2024-08-31T08:49:55.924 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 752193536 bytes 2024-08-31T08:49:56.167 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 816376832 bytes 2024-08-31T08:49:56.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 888767488 bytes 2024-08-31T08:49:56.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 903580672 bytes 2024-08-31T08:49:56.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 934031360 bytes 2024-08-31T08:49:56.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 987887616 bytes 2024-08-31T08:49:57.062 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1062315008 bytes 2024-08-31T08:49:57.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1119239168 bytes 2024-08-31T08:49:57.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1154329600 bytes 2024-08-31T08:49:57.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1166182400 bytes 2024-08-31T08:49:57.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1180102656 bytes 2024-08-31T08:49:57.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1220940800 bytes 2024-08-31T08:49:57.969 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1255759872 bytes 2024-08-31T08:49:58.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1298532352 bytes 2024-08-31T08:49:58.329 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1349434368 bytes 2024-08-31T08:49:58.469 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1374926848 bytes 2024-08-31T08:49:58.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1410002944 bytes 2024-08-31T08:49:58.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:49:58.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:49:58.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:58.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:49:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:49:58.758 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:58.803 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=112412cc203c 2024-08-31T08:49:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:49:58.804 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:49:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_2 present 112412cc203c 2024-08-31T08:49:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:49:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:58.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=112412cc203c 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:49:58.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:49:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=112412cc203c 2024-08-31T08:49:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 112412cc203c 2024-08-31T08:49:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 112412cc203c -o 112412cc203c = 112412cc203c 2024-08-31T08:49:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:49:58.949 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_2 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:49:58.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2024-08-31T08:50:03.381 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7240229352246157) > weight (0.25) 2024-08-31T08:50:03.381 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8782940244772155) > weight (0.25) 2024-08-31T08:50:03.381 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2024-08-31T08:50:03.382 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2024-08-31T08:50:03.382 INFO:tasks.rbd_mirror_thrash:waiting for 6 secs before reviving daemons 2024-08-31T08:50:03.383 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:50:03.382+0000 7f9ba0130640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 29561) UID: 1000 2024-08-31T08:50:07.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:07.489+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a949c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:50:07.491 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:07.489+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:50:09.383 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2024-08-31T08:50:09.383 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:50:09.383 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2024-08-31T08:50:09.383 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2024-08-31T08:50:09.384 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2024-08-31T08:50:09.387 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2024-08-31T08:50:09.387 INFO:tasks.rbd_mirror_thrash:waiting for 87 secs before thrashing 2024-08-31T08:50:18.298 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.28904009169464273) > weight (0.25) 2024-08-31T08:50:18.298 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5547574655762865) > weight (0.25) 2024-08-31T08:50:18.298 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.3844502073586894) > weight (0.25) 2024-08-31T08:50:18.298 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8981094635597576) > weight (0.25) 2024-08-31T08:50:18.298 INFO:tasks.rbd_mirror_thrash:waiting for 63 secs before thrashing 2024-08-31T08:50:22.405 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:22.404+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a94680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:50:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:50:29.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:50:29.946 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:50:29.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:50:29.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:50:29.983 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-08-31T08:50:29.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:50:29.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:50:29.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:50:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2024-08-31T08:50:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:50:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:50:30.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:50:30.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:50:30.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_2 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_2 2024-08-31T08:50:30.090 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2024-08-31T08:50:30.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:50:30.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_2: (2) No such file or directory 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_2 2024-08-31T08:50:30.127 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:50:30.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2024-08-31T08:50:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_2 2024-08-31T08:50:30.171 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:50:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:50:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:50:30.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_2 2024-08-31T08:50:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_2 2024-08-31T08:50:30.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2024-08-31T08:50:30.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2024-08-31T08:50:30.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:50:30.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_3 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_3 512M 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:50:30.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:50:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:50:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:50:30.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_3 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_3 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:50:30.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2024-08-31T08:50:30.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:50:30.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_3': 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 18c847e17b9f 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.18c847e17b9f 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:50:30.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:50:30 2024 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:50:30 2024 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:50:30 2024 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c3ddec03-4fa7-402e-9daf-c0d180f4d77c 2024-08-31T08:50:30.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:50:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_3 2024-08-31T08:50:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:30.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:50:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:50:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:50:30.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:50:31.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 37996544 bytes 2024-08-31T08:50:31.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 59884544 bytes 2024-08-31T08:50:31.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 108448768 bytes 2024-08-31T08:50:31.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 158395392 bytes 2024-08-31T08:50:31.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 174431232 bytes 2024-08-31T08:50:32.011 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 223997952 bytes 2024-08-31T08:50:32.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 280909824 bytes 2024-08-31T08:50:32.390 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 331002880 bytes 2024-08-31T08:50:32.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 353607680 bytes 2024-08-31T08:50:32.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 354227200 bytes 2024-08-31T08:50:32.766 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 398381056 bytes 2024-08-31T08:50:32.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 418702336 bytes 2024-08-31T08:50:33.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 445793280 bytes 2024-08-31T08:50:33.222 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 484320256 bytes 2024-08-31T08:50:33.367 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 516688896 bytes 2024-08-31T08:50:33.566 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 571216896 bytes 2024-08-31T08:50:33.650 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 571830272 bytes 2024-08-31T08:50:33.837 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 634337280 bytes 2024-08-31T08:50:34.025 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 681312256 bytes 2024-08-31T08:50:34.187 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 724038656 bytes 2024-08-31T08:50:34.494 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 740338688 bytes 2024-08-31T08:50:34.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 766848000 bytes 2024-08-31T08:50:34.749 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 781611008 bytes 2024-08-31T08:50:34.859 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 804092928 bytes 2024-08-31T08:50:35.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 840454144 bytes 2024-08-31T08:50:35.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 891996160 bytes 2024-08-31T08:50:35.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 892599296 bytes 2024-08-31T08:50:35.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 938289152 bytes 2024-08-31T08:50:35.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 938914816 bytes 2024-08-31T08:50:35.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 990428160 bytes 2024-08-31T08:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1036410880 bytes 2024-08-31T08:50:36.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1078288384 bytes 2024-08-31T08:50:36.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1119678464 bytes 2024-08-31T08:50:36.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1179466752 bytes 2024-08-31T08:50:36.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1203176448 bytes 2024-08-31T08:50:36.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1204060160 bytes 2024-08-31T08:50:36.858 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:50:36.857+0000 7fb317b8a640 -1 bdev(0x5597664cec00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device) 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fb317b8a640 time 2024-08-31T08:50:36.857993+0000 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: 664: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!") 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xc6) [0x5597622a7c07] 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:*** Caught signal (Aborted) ** 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: in thread 7fb317b8a640 thread_name:bstore_aio 2024-08-31T08:50:36.859 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:50:36.858+0000 7fb317b8a640 -1 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fb317b8a640 time 2024-08-31T08:50:36.857993+0000 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: 664: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!") 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xc6) [0x5597622a7c07] 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.860 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7fb324a3e6f0] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x8b99c) [0x7fb324a8b99c] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: raise() 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: abort() 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x5597622a7cc8] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 6: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 7: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 8: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 9: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.861 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:2024-08-31T08:50:36.859+0000 7fb317b8a640 -1 *** Caught signal (Aborted) ** 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: in thread 7fb317b8a640 thread_name:bstore_aio 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7fb324a3e6f0] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x8b99c) [0x7fb324a8b99c] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: raise() 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: abort() 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x5597622a7cc8] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 6: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 7: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 8: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 9: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T08:50:36.862 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.875 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: -158> 2024-08-31T08:50:36.857+0000 7fb317b8a640 -1 bdev(0x5597664cec00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device) 2024-08-31T08:50:36.875 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: -10> 2024-08-31T08:50:36.858+0000 7fb317b8a640 -1 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fb317b8a640 time 2024-08-31T08:50:36.857993+0000 2024-08-31T08:50:36.875 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: 664: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!") 2024-08-31T08:50:36.875 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.875 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xc6) [0x5597622a7c07] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 0> 2024-08-31T08:50:36.859+0000 7fb317b8a640 -1 *** Caught signal (Aborted) ** 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: in thread 7fb317b8a640 thread_name:bstore_aio 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7fb324a3e6f0] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x8b99c) [0x7fb324a8b99c] 2024-08-31T08:50:36.876 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: raise() 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: abort() 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x5597622a7cc8] 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 6: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 7: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 8: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 9: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T08:50:36.877 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.885 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: -315> 2024-08-31T08:50:36.857+0000 7fb317b8a640 -1 bdev(0x5597664cec00 /var/lib/ceph/osd/cluster2-0/block) _aio_thread got r=-28 ((28) No space left on device) 2024-08-31T08:50:36.885 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: -167> 2024-08-31T08:50:36.858+0000 7fb317b8a640 -1 /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: In function 'void KernelDevice::_aio_thread()' thread 7fb317b8a640 time 2024-08-31T08:50:36.857993+0000 2024-08-31T08:50:36.885 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-4620-gdc9f1ccb/rpm/el9/BUILD/ceph-19.3.0-4620-gdc9f1ccb/src/blk/kernel/KernelDevice.cc: 664: ceph_abort_msg("Unexpected IO error. This may suggest a hardware issue. Please check your kernel log!") 2024-08-31T08:50:36.885 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0xc6) [0x5597622a7c07] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: -157> 2024-08-31T08:50:36.859+0000 7fb317b8a640 -1 *** Caught signal (Aborted) ** 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: in thread 7fb317b8a640 thread_name:bstore_aio 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7fb324a3e6f0] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x8b99c) [0x7fb324a8b99c] 2024-08-31T08:50:36.886 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 3: raise() 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 4: abort() 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 5: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x187) [0x5597622a7cc8] 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 6: (KernelDevice::_aio_thread()+0x11e0) [0x559762cb4620] 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 7: ceph-osd(+0xe1c791) [0x559762cb4791] 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 8: /lib64/libc.so.6(+0x89c52) [0x7fb324a89c52] 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 9: /lib64/libc.so.6(+0x10ec80) [0x7fb324b0ec80] 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T08:50:36.887 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr: 2024-08-31T08:50:37.060 INFO:tasks.ceph.cluster2.osd.0.smithi167.stderr:daemon-helper: command failed with exit status 5 2024-08-31T08:50:37.572 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2024-08-31T08:50:38.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1273857024 bytes 2024-08-31T08:50:39.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1341392896 bytes 2024-08-31T08:50:39.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1389495296 bytes 2024-08-31T08:50:39.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1424143360 bytes 2024-08-31T08:50:39.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1447629824 bytes 2024-08-31T08:50:39.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1542279168 bytes 2024-08-31T08:50:39.678 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1567372288 bytes 2024-08-31T08:50:39.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1608094720 bytes 2024-08-31T08:50:39.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1621848064 bytes 2024-08-31T08:50:40.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1716792320 bytes 2024-08-31T08:50:40.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1761729536 bytes 2024-08-31T08:50:40.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1862699008 bytes 2024-08-31T08:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1918482432 bytes 2024-08-31T08:50:40.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1933974528 bytes 2024-08-31T08:50:40.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2013838336 bytes 2024-08-31T08:50:40.831 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2047901696 bytes 2024-08-31T08:50:40.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2052190208 bytes 2024-08-31T08:50:41.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2080229376 bytes 2024-08-31T08:50:41.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2120552448 bytes 2024-08-31T08:50:41.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2192086016 bytes 2024-08-31T08:50:41.331 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2192683008 bytes 2024-08-31T08:50:41.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2198145024 bytes 2024-08-31T08:50:41.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2255450112 bytes 2024-08-31T08:50:41.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2325163008 bytes 2024-08-31T08:50:41.854 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2393657344 bytes 2024-08-31T08:50:41.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2429946880 bytes 2024-08-31T08:50:42.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2464100352 bytes 2024-08-31T08:50:42.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2464744448 bytes 2024-08-31T08:50:42.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2525212672 bytes 2024-08-31T08:50:42.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2586258432 bytes 2024-08-31T08:50:42.524 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2610086912 bytes 2024-08-31T08:50:42.707 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2701427712 bytes 2024-08-31T08:50:42.866 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2776109056 bytes 2024-08-31T08:50:42.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2800507904 bytes 2024-08-31T08:50:42.977 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~5s 2024-08-31T08:50:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 2828585984 bytes 2024-08-31T08:50:43.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 2856472576 bytes 2024-08-31T08:50:43.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 2892130304 bytes 2024-08-31T08:50:43.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 2955328512 bytes 2024-08-31T08:50:43.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 2983519232 bytes 2024-08-31T08:50:43.594 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 2998703104 bytes 2024-08-31T08:50:43.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3067536384 bytes 2024-08-31T08:50:43.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3130630144 bytes 2024-08-31T08:50:43.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:50:43.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:50:43.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:50:43.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:50:43.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2024-08-31T08:50:43.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:50:43.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:50:43.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:50:43.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:50:43.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:50:43.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:50:43.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:50:43.991 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:50:44.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=163885d6e699 2024-08-31T08:50:44.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:50:44.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:50:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_3 present 163885d6e699 2024-08-31T08:50:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:50:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:44.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=163885d6e699 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:50:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=163885d6e699 2024-08-31T08:50:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 163885d6e699 2024-08-31T08:50:44.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 163885d6e699 -o 163885d6e699 = 163885d6e699 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_3 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:50:44.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:50:44.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2024-08-31T08:50:48.382 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~11s 2024-08-31T08:50:52.405 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5c820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:50:52.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:50:52.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/c3ddec03-4fa7-402e-9daf-c0d180f4d77c] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:50:53.787 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~16s 2024-08-31T08:50:59.191 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~22s 2024-08-31T08:51:04.595 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~27s 2024-08-31T08:51:10.000 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~32s 2024-08-31T08:51:13.733 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:51:13.731+0000 7f5ef43ce640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2f88a800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2024-08-31T08:51:13.733 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:51:13.731+0000 7f5ef43ce640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2f88a800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2024-08-31T08:51:13.757 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:51:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2024-08-31T08:51:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:13.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:51:13.763 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:51:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:51:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:51:13.764 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:13.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-08-31T08:51:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:51:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:51:13.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:51:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2024-08-31T08:51:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:51:13.900 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_3 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:13.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:13.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_3 2024-08-31T08:51:13.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2024-08-31T08:51:13.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:14.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_3: (2) No such file or directory 2024-08-31T08:51:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:51:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:51:14.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_3 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:51:14.159 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2024-08-31T08:51:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_3 2024-08-31T08:51:14.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:51:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:51:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:51:14.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_3 2024-08-31T08:51:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_3 2024-08-31T08:51:14.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2024-08-31T08:51:14.248 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2024-08-31T08:51:14.467 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_4 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_4 512M 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:14.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:14.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_4 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_4 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:51:14.520 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2024-08-31T08:51:14.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:51:14.750 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2024-08-31T08:51:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_4': 2024-08-31T08:51:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:51:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:51:14.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 18ddae6f0284 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.18ddae6f0284 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:51:14 2024 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:51:14 2024 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:51:14 2024 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 6f56ac0c-13df-4803-a7da-5f99d1121a6f 2024-08-31T08:51:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:51:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_4 2024-08-31T08:51:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:14.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:51:14.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=5 2024-08-31T08:51:14.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:51:14.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:51:15.027 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 47224832 bytes 2024-08-31T08:51:15.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 69636096 bytes 2024-08-31T08:51:15.256 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 101654528 bytes 2024-08-31T08:51:15.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 125320192 bytes 2024-08-31T08:51:15.404 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~38s 2024-08-31T08:51:15.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 156156928 bytes 2024-08-31T08:51:15.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 195445760 bytes 2024-08-31T08:51:15.643 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 237365248 bytes 2024-08-31T08:51:15.787 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 296798208 bytes 2024-08-31T08:51:15.901 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 328849408 bytes 2024-08-31T08:51:16.013 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 367858688 bytes 2024-08-31T08:51:16.189 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 442938368 bytes 2024-08-31T08:51:16.303 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 494635008 bytes 2024-08-31T08:51:16.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 530959360 bytes 2024-08-31T08:51:16.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 580146176 bytes 2024-08-31T08:51:16.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 636106752 bytes 2024-08-31T08:51:16.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 689806336 bytes 2024-08-31T08:51:17.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 818043904 bytes 2024-08-31T08:51:17.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 865651712 bytes 2024-08-31T08:51:17.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 897168384 bytes 2024-08-31T08:51:17.332 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 941845504 bytes 2024-08-31T08:51:17.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 974512128 bytes 2024-08-31T08:51:17.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1042997248 bytes 2024-08-31T08:51:17.701 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1069904896 bytes 2024-08-31T08:51:17.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1102212096 bytes 2024-08-31T08:51:17.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1127440384 bytes 2024-08-31T08:51:18.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1184288768 bytes 2024-08-31T08:51:18.165 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1224285184 bytes 2024-08-31T08:51:18.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1267593216 bytes 2024-08-31T08:51:18.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1313676288 bytes 2024-08-31T08:51:18.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1391843328 bytes 2024-08-31T08:51:18.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1446651904 bytes 2024-08-31T08:51:18.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1473789952 bytes 2024-08-31T08:51:18.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1535700992 bytes 2024-08-31T08:51:19.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1549413376 bytes 2024-08-31T08:51:19.114 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1583588352 bytes 2024-08-31T08:51:19.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1629160448 bytes 2024-08-31T08:51:19.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1677390848 bytes 2024-08-31T08:51:19.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1733582848 bytes 2024-08-31T08:51:19.575 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1745914880 bytes 2024-08-31T08:51:19.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1746514944 bytes 2024-08-31T08:51:19.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1804625920 bytes 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:19.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:19.799 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:51:19.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:51:19.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:19.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=163877be7cfe 2024-08-31T08:51:19.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:51:19.842 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:51:19.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_4 present 163877be7cfe 2024-08-31T08:51:19.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=163877be7cfe 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:51:19.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:19.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=163877be7cfe 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 163877be7cfe 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 163877be7cfe -o 163877be7cfe = 163877be7cfe 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_4 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:19.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2024-08-31T08:51:20.808 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~43s 2024-08-31T08:51:21.300 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6151638556405115) > weight (0.25) 2024-08-31T08:51:21.300 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:51:21.300 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:51:21.301 INFO:tasks.rbd_mirror_thrash:waiting for 8 secs before reviving daemons 2024-08-31T08:51:21.301 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:51:21.300+0000 7fe50bf29640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 50949) UID: 1000 2024-08-31T08:51:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:51:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:51:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:51:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/6f56ac0c-13df-4803-a7da-5f99d1121a6f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:51:26.212 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~49s 2024-08-31T08:51:29.302 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:51:29.302 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:51:29.302 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:51:29.302 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:51:29.303 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:51:29.308 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:51:29.308 INFO:tasks.rbd_mirror_thrash:waiting for 68 secs before thrashing 2024-08-31T08:51:31.615 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~54s 2024-08-31T08:51:36.388 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7844590069365345) > weight (0.25) 2024-08-31T08:51:36.388 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7097532361401105) > weight (0.25) 2024-08-31T08:51:36.388 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6601240888491376) > weight (0.25) 2024-08-31T08:51:36.388 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.9153069287740212) > weight (0.25) 2024-08-31T08:51:36.388 INFO:tasks.rbd_mirror_thrash:waiting for 83 secs before thrashing 2024-08-31T08:51:37.021 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~59s 2024-08-31T08:51:42.426 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~65s 2024-08-31T08:51:47.829 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~70s 2024-08-31T08:51:50.935 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:51:50.933+0000 7f5ef43ce640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2f011800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2024-08-31T08:51:50.935 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:51:50.933+0000 7f5ef43ce640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2f011800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2024-08-31T08:51:50.939 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:51:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2024-08-31T08:51:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:50.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:51:50.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:50.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-08-31T08:51:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:51:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:51:50.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_4 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:51:51.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:51:51.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_4 2024-08-31T08:51:51.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2024-08-31T08:51:51.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:51:51.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_4: (2) No such file or directory 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_4 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:51.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2024-08-31T08:51:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_4 2024-08-31T08:51:51.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:51:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:51:51.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_4 2024-08-31T08:51:51.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:51.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_4 2024-08-31T08:51:51.180 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2024-08-31T08:51:51.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2024-08-31T08:51:51.415 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:51:51.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:51:51.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:51:51.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_5 2024-08-31T08:51:51.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_5 512M 2024-08-31T08:51:51.419 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:51:51.420 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_5 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_5 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:51:51.468 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2024-08-31T08:51:51.954 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:51:51.962 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2024-08-31T08:51:51.978 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:51:51.977+0000 7f6cb0353640 -1 reset not still connected to 0x5599890d5e10 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_5': 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 18fe8e84f604 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.18fe8e84f604 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:51:52.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:51:51 2024 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:51:51 2024 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:51:51 2024 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 1464d851-4439-47fa-803c-e0839c0e51d1 2024-08-31T08:51:52.003 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:51:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_5 2024-08-31T08:51:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:51:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:51:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:51:52.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:51:52.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=10 2024-08-31T08:51:52.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:51:52.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:51:52.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 44387328 bytes 2024-08-31T08:51:52.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 91386880 bytes 2024-08-31T08:51:52.487 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 142859264 bytes 2024-08-31T08:51:52.591 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 167015424 bytes 2024-08-31T08:51:52.709 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 212477952 bytes 2024-08-31T08:51:52.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 240052224 bytes 2024-08-31T08:51:52.914 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 297552896 bytes 2024-08-31T08:51:53.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 354772992 bytes 2024-08-31T08:51:53.209 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 394993664 bytes 2024-08-31T08:51:53.232 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~76s 2024-08-31T08:51:53.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 439867392 bytes 2024-08-31T08:51:53.485 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 515013632 bytes 2024-08-31T08:51:53.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 572191744 bytes 2024-08-31T08:51:53.717 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 601706496 bytes 2024-08-31T08:51:53.835 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 645723136 bytes 2024-08-31T08:51:54.133 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 677765120 bytes 2024-08-31T08:51:54.223 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 703030272 bytes 2024-08-31T08:51:54.336 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 765152256 bytes 2024-08-31T08:51:54.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 816866304 bytes 2024-08-31T08:51:54.570 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 855342080 bytes 2024-08-31T08:51:54.628 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 855966720 bytes 2024-08-31T08:51:54.752 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 923940864 bytes 2024-08-31T08:51:54.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 942956544 bytes 2024-08-31T08:51:54.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 956220416 bytes 2024-08-31T08:51:55.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 985598976 bytes 2024-08-31T08:51:55.120 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1008699392 bytes 2024-08-31T08:51:55.186 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1017384960 bytes 2024-08-31T08:51:55.271 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1045368832 bytes 2024-08-31T08:51:55.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1087012864 bytes 2024-08-31T08:51:55.528 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1141253120 bytes 2024-08-31T08:51:55.612 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1161339904 bytes 2024-08-31T08:51:55.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1195037696 bytes 2024-08-31T08:51:55.809 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1230427136 bytes 2024-08-31T08:51:55.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1287846912 bytes 2024-08-31T08:51:56.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1448602624 bytes 2024-08-31T08:51:56.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1482191872 bytes 2024-08-31T08:51:56.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1556231168 bytes 2024-08-31T08:51:56.578 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1582751744 bytes 2024-08-31T08:51:56.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1598402560 bytes 2024-08-31T08:51:56.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1625105408 bytes 2024-08-31T08:51:56.903 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1677700096 bytes 2024-08-31T08:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1710534656 bytes 2024-08-31T08:51:57.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1774885888 bytes 2024-08-31T08:51:57.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1787786240 bytes 2024-08-31T08:51:57.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1827312640 bytes 2024-08-31T08:51:57.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1843656704 bytes 2024-08-31T08:51:57.601 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1911043072 bytes 2024-08-31T08:51:57.680 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1923741696 bytes 2024-08-31T08:51:57.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1941260288 bytes 2024-08-31T08:51:57.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2000737280 bytes 2024-08-31T08:51:58.023 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2029936640 bytes 2024-08-31T08:51:58.144 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2056025088 bytes 2024-08-31T08:51:58.249 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2090408960 bytes 2024-08-31T08:51:58.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2127258624 bytes 2024-08-31T08:51:58.514 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2193010688 bytes 2024-08-31T08:51:58.635 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~81s 2024-08-31T08:51:58.692 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2254932992 bytes 2024-08-31T08:51:58.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2309101568 bytes 2024-08-31T08:51:59.063 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2412578816 bytes 2024-08-31T08:51:59.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2460385280 bytes 2024-08-31T08:51:59.340 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2537721856 bytes 2024-08-31T08:51:59.532 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2586039296 bytes 2024-08-31T08:51:59.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2589959168 bytes 2024-08-31T08:51:59.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2649734144 bytes 2024-08-31T08:51:59.875 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2711173120 bytes 2024-08-31T08:51:59.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2741606400 bytes 2024-08-31T08:52:00.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2781211648 bytes 2024-08-31T08:52:00.182 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2809670656 bytes 2024-08-31T08:52:00.243 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2812619776 bytes 2024-08-31T08:52:00.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2876346368 bytes 2024-08-31T08:52:00.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2964942848 bytes 2024-08-31T08:52:00.660 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3004989440 bytes 2024-08-31T08:52:00.800 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3077031936 bytes 2024-08-31T08:52:00.906 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3100145664 bytes 2024-08-31T08:52:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3152591872 bytes 2024-08-31T08:52:01.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3231883264 bytes 2024-08-31T08:52:01.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3254825984 bytes 2024-08-31T08:52:01.429 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3287636992 bytes 2024-08-31T08:52:01.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3291520000 bytes 2024-08-31T08:52:01.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3332392960 bytes 2024-08-31T08:52:01.697 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3360957440 bytes 2024-08-31T08:52:01.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3398318080 bytes 2024-08-31T08:52:01.940 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3460319232 bytes 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:02.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:02.015 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:52:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:52:02.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=16389e9b780f 2024-08-31T08:52:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:52:02.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_5 present 16389e9b780f 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=16389e9b780f 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:52:02.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:52:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:52:02.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=16389e9b780f 2024-08-31T08:52:02.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 16389e9b780f 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 16389e9b780f -o 16389e9b780f = 16389e9b780f 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_5 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:02.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2024-08-31T08:52:04.040 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~86s 2024-08-31T08:52:09.443 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~92s 2024-08-31T08:52:14.847 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~97s 2024-08-31T08:52:20.250 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~103s 2024-08-31T08:52:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:52:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f65040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:52:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:52:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/1464d851-4439-47fa-803c-e0839c0e51d1] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:52:25.654 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~108s 2024-08-31T08:52:31.058 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~113s 2024-08-31T08:52:32.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:52:32.911+0000 7f5ef3bcd640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2fad7400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2024-08-31T08:52:32.912 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:52:32.911+0000 7f5ef3bcd640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x558b2fad7400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2024-08-31T08:52:32.916 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:52:32.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:52:32.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:52:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:52:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:52:32.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:32.968 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-08-31T08:52:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:52:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:52:32.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:52:33.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:52:33.074 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_5 2024-08-31T08:52:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_5 2024-08-31T08:52:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2024-08-31T08:52:33.075 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:33.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_5: (2) No such file or directory 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_5 2024-08-31T08:52:33.111 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:33.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_5 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:52:33.156 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:52:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_5 2024-08-31T08:52:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_5 2024-08-31T08:52:33.157 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2024-08-31T08:52:33.200 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2024-08-31T08:52:33.400 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:52:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:52:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_6 2024-08-31T08:52:33.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_6 512M 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:52:33.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_6 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_6 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:52:33.453 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2024-08-31T08:52:33.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:52:33.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2024-08-31T08:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_6': 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 1916ade52b92 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.1916ade52b92 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:52:33 2024 2024-08-31T08:52:33.976 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:52:33 2024 2024-08-31T08:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:52:33 2024 2024-08-31T08:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: c8727ab2-068f-47b6-bdb9-43ae37872a41 2024-08-31T08:52:33.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_6 2024-08-31T08:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:33.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:52:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:52:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:52:33.981 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:52:34.181 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 14706688 bytes 2024-08-31T08:52:34.268 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 15326208 bytes 2024-08-31T08:52:34.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 102939648 bytes 2024-08-31T08:52:34.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 145091584 bytes 2024-08-31T08:52:34.738 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 256246784 bytes 2024-08-31T08:52:34.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 293368832 bytes 2024-08-31T08:52:34.931 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 327246848 bytes 2024-08-31T08:52:35.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 375972864 bytes 2024-08-31T08:52:35.164 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 428473344 bytes 2024-08-31T08:52:35.220 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 429102080 bytes 2024-08-31T08:52:35.356 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 497123328 bytes 2024-08-31T08:52:35.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 549558272 bytes 2024-08-31T08:52:35.542 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 560734208 bytes 2024-08-31T08:52:35.642 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 600827904 bytes 2024-08-31T08:52:35.742 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 642651136 bytes 2024-08-31T08:52:35.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 670294016 bytes 2024-08-31T08:52:35.913 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 697080832 bytes 2024-08-31T08:52:36.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 770629632 bytes 2024-08-31T08:52:36.140 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 801547264 bytes 2024-08-31T08:52:36.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 875859968 bytes 2024-08-31T08:52:36.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 904687616 bytes 2024-08-31T08:52:36.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 939645952 bytes 2024-08-31T08:52:36.461 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~119s 2024-08-31T08:52:36.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 982019072 bytes 2024-08-31T08:52:36.632 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 995400704 bytes 2024-08-31T08:52:36.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1054067712 bytes 2024-08-31T08:52:36.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1138207744 bytes 2024-08-31T08:52:37.014 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1200945152 bytes 2024-08-31T08:52:37.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1241263104 bytes 2024-08-31T08:52:37.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1272489984 bytes 2024-08-31T08:52:37.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1306313728 bytes 2024-08-31T08:52:37.308 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5502890478744813) > weight (0.25) 2024-08-31T08:52:37.308 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:52:37.309 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:52:37.309 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-08-31T08:52:37.310 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:52:37.309+0000 7f6b9efbe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 60644) UID: 1000 2024-08-31T08:52:37.401 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1333050368 bytes 2024-08-31T08:52:37.521 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1381850112 bytes 2024-08-31T08:52:37.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1399132160 bytes 2024-08-31T08:52:37.693 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1443704832 bytes 2024-08-31T08:52:37.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1561227264 bytes 2024-08-31T08:52:37.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1599982592 bytes 2024-08-31T08:52:38.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1618385920 bytes 2024-08-31T08:52:38.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1709442048 bytes 2024-08-31T08:52:38.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1766148096 bytes 2024-08-31T08:52:38.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1824984064 bytes 2024-08-31T08:52:38.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1828326400 bytes 2024-08-31T08:52:38.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1906172928 bytes 2024-08-31T08:52:38.880 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 2008519680 bytes 2024-08-31T08:52:38.951 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 2052091904 bytes 2024-08-31T08:52:39.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 2117313536 bytes 2024-08-31T08:52:39.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 2153565184 bytes 2024-08-31T08:52:39.294 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2190978048 bytes 2024-08-31T08:52:39.310 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:52:39.311 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:52:39.311 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:52:39.311 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:52:39.311 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:52:39.314 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:52:39.314 INFO:tasks.rbd_mirror_thrash:waiting for 82 secs before thrashing 2024-08-31T08:52:39.384 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2213728256 bytes 2024-08-31T08:52:39.474 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2230145024 bytes 2024-08-31T08:52:39.554 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2250526720 bytes 2024-08-31T08:52:39.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2287607808 bytes 2024-08-31T08:52:39.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2336478208 bytes 2024-08-31T08:52:39.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2351078400 bytes 2024-08-31T08:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:39.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:39.987 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:52:39.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:40.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=16385c8e8dac 2024-08-31T08:52:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:52:40.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_6 present 16385c8e8dac 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=16385c8e8dac 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:52:40.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:52:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:52:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:52:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:52:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:52:40.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:52:40.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=16385c8e8dac 2024-08-31T08:52:40.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 16385c8e8dac 2024-08-31T08:52:40.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 16385c8e8dac -o 16385c8e8dac = 16385c8e8dac 2024-08-31T08:52:40.177 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_6 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:52:40.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2024-08-31T08:52:41.866 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~124s 2024-08-31T08:52:47.270 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~130s 2024-08-31T08:52:52.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:52:52.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:52:52.407 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:52:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:52:52.674 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~135s 2024-08-31T08:52:58.078 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~141s 2024-08-31T08:52:59.389 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8736830010482489) > weight (0.25) 2024-08-31T08:52:59.389 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2024-08-31T08:52:59.389 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2024-08-31T08:52:59.390 INFO:tasks.rbd_mirror_thrash:waiting for 7 secs before reviving daemons 2024-08-31T08:52:59.390 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.smithi167.stderr:2024-08-31T08:52:59.390+0000 7f5f01be9640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 54993) UID: 1000 2024-08-31T08:53:03.482 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~146s 2024-08-31T08:53:03.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:53:03.487+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f62680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:53:03.488 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:53:03.487+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:53:06.391 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2024-08-31T08:53:06.391 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:53:06.391 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2024-08-31T08:53:06.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2024-08-31T08:53:06.392 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2024-08-31T08:53:06.395 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2024-08-31T08:53:06.395 INFO:tasks.rbd_mirror_thrash:waiting for 109 secs before thrashing 2024-08-31T08:53:08.886 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~151s 2024-08-31T08:53:10.909 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:53:10.927 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:53:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:53:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:53:10.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:10.963 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-08-31T08:53:10.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:53:10.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:53:10.966 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:53:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2024-08-31T08:53:11.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_6 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_6 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2024-08-31T08:53:11.069 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:11.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_6: (2) No such file or directory 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_6 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:53:11.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2024-08-31T08:53:11.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_6 2024-08-31T08:53:11.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:53:11.153 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:53:11.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:53:11.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_6 2024-08-31T08:53:11.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.154 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_6 2024-08-31T08:53:11.155 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2024-08-31T08:53:11.197 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2024-08-31T08:53:11.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_7 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_7 512M 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:11.392 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:11.393 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_7 2024-08-31T08:53:11.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_7 2024-08-31T08:53:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:53:11.441 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2024-08-31T08:53:11.928 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:53:11.937 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_7': 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 193d2847ff8a 2024-08-31T08:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.193d2847ff8a 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:53:11 2024 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:53:11 2024 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:53:11 2024 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: ea665289-b2d3-4e6c-b5fa-e7e081983ada 2024-08-31T08:53:11.975 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:53:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_7 2024-08-31T08:53:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:11.978 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:53:11.979 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=8 2024-08-31T08:53:11.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:53:11.980 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:53:12.199 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 28239872 bytes 2024-08-31T08:53:12.305 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 47918080 bytes 2024-08-31T08:53:12.388 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 64824320 bytes 2024-08-31T08:53:12.502 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 105600000 bytes 2024-08-31T08:53:12.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 165432320 bytes 2024-08-31T08:53:12.753 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 237758464 bytes 2024-08-31T08:53:12.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 288748544 bytes 2024-08-31T08:53:12.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 350091264 bytes 2024-08-31T08:53:13.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 394955776 bytes 2024-08-31T08:53:13.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 395555840 bytes 2024-08-31T08:53:13.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 418248704 bytes 2024-08-31T08:53:13.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 508025856 bytes 2024-08-31T08:53:13.534 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 604301312 bytes 2024-08-31T08:53:13.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 690574336 bytes 2024-08-31T08:53:13.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 746259456 bytes 2024-08-31T08:53:13.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 747324416 bytes 2024-08-31T08:53:13.996 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 793044992 bytes 2024-08-31T08:53:14.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 865940480 bytes 2024-08-31T08:53:14.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 890037248 bytes 2024-08-31T08:53:14.290 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~157s 2024-08-31T08:53:14.383 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 987115520 bytes 2024-08-31T08:53:14.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 987701248 bytes 2024-08-31T08:53:14.559 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 1023436800 bytes 2024-08-31T08:53:14.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1114980352 bytes 2024-08-31T08:53:14.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1126000640 bytes 2024-08-31T08:53:14.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1144626176 bytes 2024-08-31T08:53:14.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1165298688 bytes 2024-08-31T08:53:15.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1194793984 bytes 2024-08-31T08:53:15.142 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1234519040 bytes 2024-08-31T08:53:15.241 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1268509696 bytes 2024-08-31T08:53:15.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1281591296 bytes 2024-08-31T08:53:15.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1304199168 bytes 2024-08-31T08:53:15.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1339001856 bytes 2024-08-31T08:53:15.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1373484032 bytes 2024-08-31T08:53:15.702 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1415053312 bytes 2024-08-31T08:53:15.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1458163712 bytes 2024-08-31T08:53:15.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1468419072 bytes 2024-08-31T08:53:16.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1569397760 bytes 2024-08-31T08:53:16.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1583087616 bytes 2024-08-31T08:53:16.250 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1661776896 bytes 2024-08-31T08:53:16.376 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1716738048 bytes 2024-08-31T08:53:16.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1744591872 bytes 2024-08-31T08:53:16.631 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1824335872 bytes 2024-08-31T08:53:16.746 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1878686720 bytes 2024-08-31T08:53:16.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1879294976 bytes 2024-08-31T08:53:16.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1923680256 bytes 2024-08-31T08:53:17.061 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1964148736 bytes 2024-08-31T08:53:17.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2004615168 bytes 2024-08-31T08:53:17.267 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2050285568 bytes 2024-08-31T08:53:17.359 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2080375808 bytes 2024-08-31T08:53:17.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2129317888 bytes 2024-08-31T08:53:17.564 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2170948608 bytes 2024-08-31T08:53:17.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2291693568 bytes 2024-08-31T08:53:17.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2311199744 bytes 2024-08-31T08:53:18.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2372173824 bytes 2024-08-31T08:53:18.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2406677504 bytes 2024-08-31T08:53:18.233 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2453600256 bytes 2024-08-31T08:53:18.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2509053952 bytes 2024-08-31T08:53:18.491 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2551829504 bytes 2024-08-31T08:53:18.613 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2605007872 bytes 2024-08-31T08:53:18.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2640474112 bytes 2024-08-31T08:53:18.781 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2641070080 bytes 2024-08-31T08:53:18.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2712919040 bytes 2024-08-31T08:53:19.038 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2741588992 bytes 2024-08-31T08:53:19.126 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2763664384 bytes 2024-08-31T08:53:19.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2825913344 bytes 2024-08-31T08:53:19.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2874398720 bytes 2024-08-31T08:53:19.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2933596160 bytes 2024-08-31T08:53:19.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2955310080 bytes 2024-08-31T08:53:19.685 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2966850560 bytes 2024-08-31T08:53:19.692 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~162s 2024-08-31T08:53:19.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3001996288 bytes 2024-08-31T08:53:19.930 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3069040640 bytes 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:19.984 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:19.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:19.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:53:19.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:53:19.986 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=178babb14efa 2024-08-31T08:53:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:53:20.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:53:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_7 present 178babb14efa 2024-08-31T08:53:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:20.130 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=178babb14efa 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:53:20.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=178babb14efa 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 178babb14efa 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 178babb14efa -o 178babb14efa = 178babb14efa 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:53:20.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_7 2024-08-31T08:53:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:20.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2024-08-31T08:53:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:53:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3735040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:53:22.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:53:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/ea665289-b2d3-4e6c-b5fa-e7e081983ada] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:53:25.096 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~168s 2024-08-31T08:53:30.500 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~173s 2024-08-31T08:53:35.904 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~178s 2024-08-31T08:53:41.308 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~184s 2024-08-31T08:53:46.713 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~189s 2024-08-31T08:53:51.783 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:53:51.781+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5d578400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2024-08-31T08:53:51.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:53:51.781+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5d578400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=192, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2024-08-31T08:53:51.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:53:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2024-08-31T08:53:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:51.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:53:51.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:51.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-08-31T08:53:51.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:53:51.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:53:51.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:53:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2024-08-31T08:53:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:53:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:51.952 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_7 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_7 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2024-08-31T08:53:51.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:53:51.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_7: (2) No such file or directory 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_7 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:51.992 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:51.993 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2024-08-31T08:53:52.035 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_7 2024-08-31T08:53:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:53:52.036 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:53:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_7 2024-08-31T08:53:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_7 2024-08-31T08:53:52.037 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2024-08-31T08:53:52.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2024-08-31T08:53:52.116 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~195s 2024-08-31T08:53:52.284 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:53:52.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_8 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_8 512M 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:52.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:53:52.292 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:53:52.293 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_8 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_8 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:53:52.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2024-08-31T08:53:52.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:53:52.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_8': 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 195567fe8de7 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.195567fe8de7 2024-08-31T08:53:52.843 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:53:52 2024 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:53:52 2024 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:53:52 2024 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:53:52.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:53:52.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 900e5810-e68f-4926-b489-2be949a4a59b 2024-08-31T08:53:52.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:53:52.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_8 2024-08-31T08:53:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:53:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:53:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:53:52.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:53:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=14 2024-08-31T08:53:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:53:52.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:53:53.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 16239616 bytes 2024-08-31T08:53:53.208 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 72387584 bytes 2024-08-31T08:53:53.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 122477568 bytes 2024-08-31T08:53:53.460 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 160221184 bytes 2024-08-31T08:53:53.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 213271552 bytes 2024-08-31T08:53:53.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 270302208 bytes 2024-08-31T08:53:53.785 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 292541440 bytes 2024-08-31T08:53:53.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 341815296 bytes 2024-08-31T08:53:54.033 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 407481344 bytes 2024-08-31T08:53:54.198 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 479543296 bytes 2024-08-31T08:53:54.308 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 504729600 bytes 2024-08-31T08:53:54.398 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 544497664 bytes 2024-08-31T08:53:54.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 584311808 bytes 2024-08-31T08:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 666516480 bytes 2024-08-31T08:53:54.822 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 745357312 bytes 2024-08-31T08:53:54.925 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 778728448 bytes 2024-08-31T08:53:55.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 798727168 bytes 2024-08-31T08:53:55.086 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 812554240 bytes 2024-08-31T08:53:55.172 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 846520320 bytes 2024-08-31T08:53:55.264 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 868503552 bytes 2024-08-31T08:53:55.349 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 892252160 bytes 2024-08-31T08:53:55.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 947323904 bytes 2024-08-31T08:53:55.670 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 1037556736 bytes 2024-08-31T08:53:55.784 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1081778176 bytes 2024-08-31T08:53:55.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1082405888 bytes 2024-08-31T08:53:55.899 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1087664128 bytes 2024-08-31T08:53:56.039 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1136065536 bytes 2024-08-31T08:53:56.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1176242176 bytes 2024-08-31T08:53:56.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1276022784 bytes 2024-08-31T08:53:56.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1304082432 bytes 2024-08-31T08:53:56.536 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1340005376 bytes 2024-08-31T08:53:56.625 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1380928512 bytes 2024-08-31T08:53:56.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1434149888 bytes 2024-08-31T08:53:56.908 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1487726592 bytes 2024-08-31T08:53:57.009 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1524771840 bytes 2024-08-31T08:53:57.091 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1576135680 bytes 2024-08-31T08:53:57.231 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1597641728 bytes 2024-08-31T08:53:57.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1632921600 bytes 2024-08-31T08:53:57.464 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1676856320 bytes 2024-08-31T08:53:57.519 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~200s 2024-08-31T08:53:57.580 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1715017728 bytes 2024-08-31T08:53:57.706 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1766450176 bytes 2024-08-31T08:53:57.815 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1810853888 bytes 2024-08-31T08:53:57.929 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1849164800 bytes 2024-08-31T08:53:58.032 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1879169024 bytes 2024-08-31T08:53:58.210 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1956811776 bytes 2024-08-31T08:53:58.285 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1974193152 bytes 2024-08-31T08:53:58.378 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1994649600 bytes 2024-08-31T08:53:58.558 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2119161856 bytes 2024-08-31T08:53:58.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2121451520 bytes 2024-08-31T08:53:58.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2152181760 bytes 2024-08-31T08:53:58.867 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2167194624 bytes 2024-08-31T08:53:58.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2188565504 bytes 2024-08-31T08:53:59.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2258997248 bytes 2024-08-31T08:53:59.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2315074560 bytes 2024-08-31T08:53:59.322 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2315695104 bytes 2024-08-31T08:53:59.426 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2345221120 bytes 2024-08-31T08:53:59.595 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2430199808 bytes 2024-08-31T08:53:59.653 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2431473664 bytes 2024-08-31T08:53:59.762 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2473463808 bytes 2024-08-31T08:53:59.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2511795200 bytes 2024-08-31T08:53:59.977 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2534586368 bytes 2024-08-31T08:54:00.104 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2581395456 bytes 2024-08-31T08:54:00.273 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2667558912 bytes 2024-08-31T08:54:00.421 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2738153472 bytes 2024-08-31T08:54:00.562 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2776136704 bytes 2024-08-31T08:54:00.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2784751616 bytes 2024-08-31T08:54:00.829 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2866077696 bytes 2024-08-31T08:54:00.935 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2891742208 bytes 2024-08-31T08:54:01.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2918638592 bytes 2024-08-31T08:54:01.128 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 2945242112 bytes 2024-08-31T08:54:01.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3014045696 bytes 2024-08-31T08:54:01.315 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4948017491327932) > weight (0.25) 2024-08-31T08:54:01.316 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.44971555797084883) > weight (0.25) 2024-08-31T08:54:01.316 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8044210447364777) > weight (0.25) 2024-08-31T08:54:01.316 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2024-08-31T08:54:01.316 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2024-08-31T08:54:01.317 INFO:tasks.rbd_mirror_thrash:waiting for 1 secs before reviving daemons 2024-08-31T08:54:01.317 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.smithi167.stderr:2024-08-31T08:54:01.317+0000 7ff37960b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 31611) UID: 1000 2024-08-31T08:54:01.428 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3053400064 bytes 2024-08-31T08:54:01.531 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3088120832 bytes 2024-08-31T08:54:01.661 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3134911488 bytes 2024-08-31T08:54:01.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3238731776 bytes 2024-08-31T08:54:01.973 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3298233344 bytes 2024-08-31T08:54:02.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3306342400 bytes 2024-08-31T08:54:02.169 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3341247488 bytes 2024-08-31T08:54:02.276 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3379959808 bytes 2024-08-31T08:54:02.318 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2024-08-31T08:54:02.318 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:54:02.318 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2024-08-31T08:54:02.319 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2024-08-31T08:54:02.319 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2024-08-31T08:54:02.322 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2024-08-31T08:54:02.322 INFO:tasks.rbd_mirror_thrash:waiting for 81 secs before thrashing 2024-08-31T08:54:02.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3424990208 bytes 2024-08-31T08:54:02.527 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3455797248 bytes 2024-08-31T08:54:02.686 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8200 IOs, 3523269632 bytes 2024-08-31T08:54:02.777 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8300 IOs, 3555087360 bytes 2024-08-31T08:54:02.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8400 IOs, 3615283200 bytes 2024-08-31T08:54:02.922 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~205s 2024-08-31T08:54:02.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8500 IOs, 3621738496 bytes 2024-08-31T08:54:03.089 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8600 IOs, 3647454208 bytes 2024-08-31T08:54:03.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8700 IOs, 3681568768 bytes 2024-08-31T08:54:03.339 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8800 IOs, 3741231104 bytes 2024-08-31T08:54:03.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8900 IOs, 3785298944 bytes 2024-08-31T08:54:03.522 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9000 IOs, 3798049792 bytes 2024-08-31T08:54:03.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9100 IOs, 3860076544 bytes 2024-08-31T08:54:03.879 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9200 IOs, 3984187392 bytes 2024-08-31T08:54:04.077 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9300 IOs, 4062377984 bytes 2024-08-31T08:54:04.185 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9400 IOs, 4093172736 bytes 2024-08-31T08:54:04.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9500 IOs, 4126871552 bytes 2024-08-31T08:54:04.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9600 IOs, 4168078336 bytes 2024-08-31T08:54:04.553 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9700 IOs, 4225778688 bytes 2024-08-31T08:54:04.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9800 IOs, 4292956160 bytes 2024-08-31T08:54:04.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9900 IOs, 4327622656 bytes 2024-08-31T08:54:04.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10000 IOs, 4354471936 bytes 2024-08-31T08:54:04.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10100 IOs, 4364832768 bytes 2024-08-31T08:54:05.106 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10200 IOs, 4420364288 bytes 2024-08-31T08:54:05.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10300 IOs, 4445423616 bytes 2024-08-31T08:54:05.396 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10400 IOs, 4555385856 bytes 2024-08-31T08:54:05.489 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10500 IOs, 4589924352 bytes 2024-08-31T08:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10600 IOs, 4612688896 bytes 2024-08-31T08:54:05.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10700 IOs, 4655306752 bytes 2024-08-31T08:54:05.798 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10800 IOs, 4685586432 bytes 2024-08-31T08:54:05.933 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10900 IOs, 4733876224 bytes 2024-08-31T08:54:06.068 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11000 IOs, 4780356608 bytes 2024-08-31T08:54:06.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11100 IOs, 4839628800 bytes 2024-08-31T08:54:06.330 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11200 IOs, 4890223616 bytes 2024-08-31T08:54:06.389 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11300 IOs, 4890839040 bytes 2024-08-31T08:54:06.523 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11400 IOs, 4943439872 bytes 2024-08-31T08:54:06.629 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11500 IOs, 4985892864 bytes 2024-08-31T08:54:06.741 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:11600 IOs, 5036785664 bytes 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:54:06.857 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:54:06.858 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:06.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=178b4aa3c010 2024-08-31T08:54:06.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:54:06.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_8 present 178b4aa3c010 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=178b4aa3c010 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:54:07.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:54:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:54:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:54:07.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:07.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=178b4aa3c010 2024-08-31T08:54:07.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 178b4aa3c010 2024-08-31T08:54:07.050 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 178b4aa3c010 -o 178b4aa3c010 = 178b4aa3c010 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_8 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:07.051 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2024-08-31T08:54:08.325 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~211s 2024-08-31T08:54:13.730 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~216s 2024-08-31T08:54:19.133 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~222s 2024-08-31T08:54:22.407 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:54:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:54:22.407 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:54:22.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/900e5810-e68f-4926-b489-2be949a4a59b] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:54:24.537 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~227s 2024-08-31T08:54:29.942 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~232s 2024-08-31T08:54:35.346 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~238s 2024-08-31T08:54:36.942 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:54:36.941+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5dbf0000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=192 2024-08-31T08:54:36.943 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:54:36.941+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5dbf0000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=193, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=c0, last_copied_object_number=128, snap_seqs={192=18446744073709551614}] 2024-08-31T08:54:36.950 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:36.964 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:54:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:54:36.965 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:37.002 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-08-31T08:54:37.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:54:37.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:54:37.005 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:54:37.107 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_8 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:37.108 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_8 2024-08-31T08:54:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2024-08-31T08:54:37.109 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:37.143 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_8: (2) No such file or directory 2024-08-31T08:54:37.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:54:37.146 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_8 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:37.147 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_8 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:54:37.192 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:54:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_8 2024-08-31T08:54:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_8 2024-08-31T08:54:37.193 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2024-08-31T08:54:37.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2024-08-31T08:54:37.447 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:54:37.450 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_9 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_9 512M 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:54:37.451 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:54:37.452 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_9 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_9 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:54:37.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2024-08-31T08:54:37.960 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:54:37.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_9': 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 197667775113 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.197667775113 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:54:38.007 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:54:37 2024 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:54:37 2024 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:54:37 2024 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: d109198c-6282-42c0-8b36-959ff5918e18 2024-08-31T08:54:38.008 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:54:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_9 2024-08-31T08:54:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:38.010 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:54:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=6 2024-08-31T08:54:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:54:38.012 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:54:38.224 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 17597440 bytes 2024-08-31T08:54:38.317 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 20664320 bytes 2024-08-31T08:54:38.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 93259776 bytes 2024-08-31T08:54:38.611 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 167748608 bytes 2024-08-31T08:54:38.746 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 227384320 bytes 2024-08-31T08:54:38.877 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 287766528 bytes 2024-08-31T08:54:38.988 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 327561216 bytes 2024-08-31T08:54:39.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 354653184 bytes 2024-08-31T08:54:39.152 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 356972544 bytes 2024-08-31T08:54:39.261 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 393048064 bytes 2024-08-31T08:54:39.371 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 440073216 bytes 2024-08-31T08:54:39.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 525796352 bytes 2024-08-31T08:54:39.622 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 546188288 bytes 2024-08-31T08:54:39.712 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 561500160 bytes 2024-08-31T08:54:39.872 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 632656896 bytes 2024-08-31T08:54:39.948 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 652187648 bytes 2024-08-31T08:54:40.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 702759936 bytes 2024-08-31T08:54:40.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 723511296 bytes 2024-08-31T08:54:40.286 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 776904704 bytes 2024-08-31T08:54:40.386 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 820982784 bytes 2024-08-31T08:54:40.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 868950016 bytes 2024-08-31T08:54:40.603 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 896572416 bytes 2024-08-31T08:54:40.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 936304640 bytes 2024-08-31T08:54:40.750 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~243s 2024-08-31T08:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 971795456 bytes 2024-08-31T08:54:40.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1038507008 bytes 2024-08-31T08:54:41.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1088716800 bytes 2024-08-31T08:54:41.158 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1132715008 bytes 2024-08-31T08:54:41.244 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1158385664 bytes 2024-08-31T08:54:41.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1198145536 bytes 2024-08-31T08:54:41.465 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1232760832 bytes 2024-08-31T08:54:41.587 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1291430912 bytes 2024-08-31T08:54:41.649 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1292046336 bytes 2024-08-31T08:54:41.728 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1329117184 bytes 2024-08-31T08:54:41.813 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1341209600 bytes 2024-08-31T08:54:41.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1346744320 bytes 2024-08-31T08:54:41.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1387324416 bytes 2024-08-31T08:54:42.041 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1402137600 bytes 2024-08-31T08:54:42.145 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1432903680 bytes 2024-08-31T08:54:42.328 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1533836288 bytes 2024-08-31T08:54:42.414 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1562072064 bytes 2024-08-31T08:54:42.496 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1575952384 bytes 2024-08-31T08:54:42.637 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1644755968 bytes 2024-08-31T08:54:42.720 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1685777408 bytes 2024-08-31T08:54:42.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1726007296 bytes 2024-08-31T08:54:42.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1782898688 bytes 2024-08-31T08:54:43.049 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1783501824 bytes 2024-08-31T08:54:43.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 1838374912 bytes 2024-08-31T08:54:43.236 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 1838971904 bytes 2024-08-31T08:54:43.341 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 1879140352 bytes 2024-08-31T08:54:43.425 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 1900647424 bytes 2024-08-31T08:54:43.499 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 1942555648 bytes 2024-08-31T08:54:43.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 1963058176 bytes 2024-08-31T08:54:43.677 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 1972694016 bytes 2024-08-31T08:54:43.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 1973324800 bytes 2024-08-31T08:54:43.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2024756224 bytes 2024-08-31T08:54:43.974 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2071939072 bytes 2024-08-31T08:54:44.016 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:54:44.017 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:54:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:54:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:54:44.018 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:44.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=178b75becb22 2024-08-31T08:54:44.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:54:44.059 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:54:44.160 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_9 present 178b75becb22 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=178b75becb22 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:54:44.161 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:54:44.162 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:54:44.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=178b75becb22 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 178b75becb22 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 178b75becb22 -o 178b75becb22 = 178b75becb22 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_9 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:54:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2024-08-31T08:54:46.154 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~249s 2024-08-31T08:54:51.558 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~254s 2024-08-31T08:54:52.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:54:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5b380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T08:54:52.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:54:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/d109198c-6282-42c0-8b36-959ff5918e18] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T08:54:55.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7277568975721863) > weight (0.25) 2024-08-31T08:54:55.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5025882939364035) > weight (0.25) 2024-08-31T08:54:55.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8160076615590807) > weight (0.25) 2024-08-31T08:54:55.396 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.654480138697448) > weight (0.25) 2024-08-31T08:54:55.397 INFO:tasks.rbd_mirror_thrash:waiting for 92 secs before thrashing 2024-08-31T08:54:56.961 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~259s 2024-08-31T08:55:02.365 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~265s 2024-08-31T08:55:07.769 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~270s 2024-08-31T08:55:13.174 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~276s 2024-08-31T08:55:15.826 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:55:15.824+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5d5d6400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2024-08-31T08:55:15.826 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.smithi167.stderr:2024-08-31T08:55:15.824+0000 7fb29520c640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x561a5d5d6400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=b2ed8de6-469c-4293-9d02-39498e569bf2, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2024-08-31T08:55:15.830 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring disabled 2024-08-31T08:55:15.844 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=deleted 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:55:15.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:55:15.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:55:15.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-08-31T08:55:15.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id= 2024-08-31T08:55:15.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:55:15.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=deleted 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:55:15.989 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_9 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_9 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2024-08-31T08:55:15.990 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:55:16.024 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: error opening image image_9: (2) No such file or directory 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id= 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test deleted = deleted 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ purge_snapshots cluster2 mirror image_9 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:55:16.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2024-08-31T08:55:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image_retry cluster2 mirror image_9 2024-08-31T08:55:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.071 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:55:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0 1 2 4 8 16 32 2024-08-31T08:55:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0 2024-08-31T08:55:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ remove_image cluster2 mirror image_9 2024-08-31T08:55:16.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_9 2024-08-31T08:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2024-08-31T08:55:16.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2024-08-31T08:55:16.321 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-08-31T08:55:16.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for i in `seq 1 10` 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image=image_10 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ create_image cluster2 mirror image_10 512M 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:55:16.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local size=128 2024-08-31T08:55:16.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' -n 512M ']' 2024-08-31T08:55:16.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ size=512M 2024-08-31T08:55:16.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ shift 2024-08-31T08:55:16.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_10 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ enable_mirror cluster2 mirror image_10 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local mode=snapshot 2024-08-31T08:55:16.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2024-08-31T08:55:16.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirroring enabled 2024-08-31T08:55:16.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2024-08-31T08:55:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_10': 2024-08-31T08:55:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T08:55:16.886 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 198ee34a18f2 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.198ee34a18f2 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:55:16 2024 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:55:16 2024 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:55:16 2024 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T08:55:16.887 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T08:55:16.888 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: true 2024-08-31T08:55:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ stress_write_image cluster2 mirror image_10 2024-08-31T08:55:16.889 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:16.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2024-08-31T08:55:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local duration=13 2024-08-31T08:55:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set +e 2024-08-31T08:55:16.891 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2024-08-31T08:55:17.080 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:100 IOs, 4560896 bytes 2024-08-31T08:55:17.229 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:200 IOs, 55987200 bytes 2024-08-31T08:55:17.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:300 IOs, 182168576 bytes 2024-08-31T08:55:17.602 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:400 IOs, 232461312 bytes 2024-08-31T08:55:17.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:500 IOs, 268749824 bytes 2024-08-31T08:55:17.797 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:600 IOs, 288296960 bytes 2024-08-31T08:55:17.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:700 IOs, 358618112 bytes 2024-08-31T08:55:18.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:800 IOs, 402926592 bytes 2024-08-31T08:55:18.190 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:900 IOs, 469645312 bytes 2024-08-31T08:55:18.395 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1000 IOs, 559890432 bytes 2024-08-31T08:55:18.480 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1100 IOs, 581025792 bytes 2024-08-31T08:55:18.577 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~281s 2024-08-31T08:55:18.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1200 IOs, 640831488 bytes 2024-08-31T08:55:18.690 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1300 IOs, 642612224 bytes 2024-08-31T08:55:18.778 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1400 IOs, 672722944 bytes 2024-08-31T08:55:19.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1500 IOs, 723766272 bytes 2024-08-31T08:55:19.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1600 IOs, 790003712 bytes 2024-08-31T08:55:19.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1700 IOs, 790592512 bytes 2024-08-31T08:55:19.462 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1800 IOs, 830183424 bytes 2024-08-31T08:55:19.592 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:1900 IOs, 881094656 bytes 2024-08-31T08:55:19.703 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2000 IOs, 936151040 bytes 2024-08-31T08:55:19.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2100 IOs, 952305664 bytes 2024-08-31T08:55:19.890 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2200 IOs, 960935936 bytes 2024-08-31T08:55:19.995 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2300 IOs, 989782016 bytes 2024-08-31T08:55:20.093 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2400 IOs, 1030998016 bytes 2024-08-31T08:55:20.191 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2500 IOs, 1066486784 bytes 2024-08-31T08:55:20.348 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2600 IOs, 1114029056 bytes 2024-08-31T08:55:20.459 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2700 IOs, 1148163072 bytes 2024-08-31T08:55:20.560 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2800 IOs, 1182901248 bytes 2024-08-31T08:55:20.666 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:2900 IOs, 1239011328 bytes 2024-08-31T08:55:20.779 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3000 IOs, 1246914560 bytes 2024-08-31T08:55:20.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3100 IOs, 1333225472 bytes 2024-08-31T08:55:21.070 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3200 IOs, 1356145664 bytes 2024-08-31T08:55:21.247 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3300 IOs, 1446091776 bytes 2024-08-31T08:55:21.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3400 IOs, 1482924032 bytes 2024-08-31T08:55:21.463 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3500 IOs, 1505981440 bytes 2024-08-31T08:55:21.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3600 IOs, 1592731648 bytes 2024-08-31T08:55:21.713 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3700 IOs, 1593326592 bytes 2024-08-31T08:55:21.796 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3800 IOs, 1615737856 bytes 2024-08-31T08:55:21.953 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:3900 IOs, 1670179840 bytes 2024-08-31T08:55:22.112 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4000 IOs, 1738979328 bytes 2024-08-31T08:55:22.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4100 IOs, 1773036544 bytes 2024-08-31T08:55:22.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4200 IOs, 1779969024 bytes 2024-08-31T08:55:22.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4300 IOs, 1852144640 bytes 2024-08-31T08:55:22.555 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4400 IOs, 1877831680 bytes 2024-08-31T08:55:22.691 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4500 IOs, 1927468032 bytes 2024-08-31T08:55:22.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4600 IOs, 1982199808 bytes 2024-08-31T08:55:22.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4700 IOs, 2012146688 bytes 2024-08-31T08:55:23.054 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4800 IOs, 2047744000 bytes 2024-08-31T08:55:23.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:4900 IOs, 2097035264 bytes 2024-08-31T08:55:23.322 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4744135218775045) > weight (0.25) 2024-08-31T08:55:23.323 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2024-08-31T08:55:23.323 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2024-08-31T08:55:23.323 INFO:tasks.rbd_mirror_thrash:waiting for 5 secs before reviving daemons 2024-08-31T08:55:23.324 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5000 IOs, 2172836864 bytes 2024-08-31T08:55:23.324 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.smithi167.stderr:2024-08-31T08:55:23.322+0000 7f6264bcb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 61187) UID: 1000 2024-08-31T08:55:23.439 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5100 IOs, 2195233792 bytes 2024-08-31T08:55:23.518 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5200 IOs, 2206304256 bytes 2024-08-31T08:55:23.669 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5300 IOs, 2279110656 bytes 2024-08-31T08:55:23.819 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5400 IOs, 2335915008 bytes 2024-08-31T08:55:23.938 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5500 IOs, 2388190208 bytes 2024-08-31T08:55:23.981 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~286s 2024-08-31T08:55:24.028 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5600 IOs, 2417910784 bytes 2024-08-31T08:55:24.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5700 IOs, 2418531328 bytes 2024-08-31T08:55:24.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5800 IOs, 2461577216 bytes 2024-08-31T08:55:24.399 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:5900 IOs, 2547816448 bytes 2024-08-31T08:55:24.525 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6000 IOs, 2606751744 bytes 2024-08-31T08:55:24.648 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6100 IOs, 2644317184 bytes 2024-08-31T08:55:24.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6200 IOs, 2707190784 bytes 2024-08-31T08:55:24.923 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6300 IOs, 2755028992 bytes 2024-08-31T08:55:25.072 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6400 IOs, 2808806400 bytes 2024-08-31T08:55:25.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6500 IOs, 2868367360 bytes 2024-08-31T08:55:25.277 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6600 IOs, 2890007552 bytes 2024-08-31T08:55:25.412 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6700 IOs, 2928423936 bytes 2024-08-31T08:55:25.514 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6800 IOs, 2953149440 bytes 2024-08-31T08:55:25.600 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:6900 IOs, 2977474560 bytes 2024-08-31T08:55:25.756 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7000 IOs, 3043692544 bytes 2024-08-31T08:55:25.876 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7100 IOs, 3088220160 bytes 2024-08-31T08:55:25.971 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7200 IOs, 3112843264 bytes 2024-08-31T08:55:26.078 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7300 IOs, 3153476608 bytes 2024-08-31T08:55:26.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7400 IOs, 3169147904 bytes 2024-08-31T08:55:26.257 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7500 IOs, 3197816832 bytes 2024-08-31T08:55:26.416 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7600 IOs, 3263382528 bytes 2024-08-31T08:55:26.538 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7700 IOs, 3280302080 bytes 2024-08-31T08:55:26.597 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7800 IOs, 3282623488 bytes 2024-08-31T08:55:26.696 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:7900 IOs, 3311733760 bytes 2024-08-31T08:55:26.823 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8000 IOs, 3368536064 bytes 2024-08-31T08:55:26.970 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8100 IOs, 3428116480 bytes 2024-08-31T08:55:27.040 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8200 IOs, 3428733952 bytes 2024-08-31T08:55:27.123 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8300 IOs, 3450537984 bytes 2024-08-31T08:55:27.228 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8400 IOs, 3486601216 bytes 2024-08-31T08:55:27.377 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8500 IOs, 3550169088 bytes 2024-08-31T08:55:27.461 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8600 IOs, 3580941312 bytes 2024-08-31T08:55:27.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8700 IOs, 3593351168 bytes 2024-08-31T08:55:27.621 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8800 IOs, 3593981952 bytes 2024-08-31T08:55:27.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:8900 IOs, 3606618112 bytes 2024-08-31T08:55:27.782 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9000 IOs, 3623299072 bytes 2024-08-31T08:55:27.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9100 IOs, 3664783360 bytes 2024-08-31T08:55:27.994 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9200 IOs, 3699133440 bytes 2024-08-31T08:55:28.131 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9300 IOs, 3753613312 bytes 2024-08-31T08:55:28.246 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9400 IOs, 3788157952 bytes 2024-08-31T08:55:28.324 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2024-08-31T08:55:28.324 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:55:28.325 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2024-08-31T08:55:28.325 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2024-08-31T08:55:28.325 DEBUG:teuthology.orchestra.run.smithi167:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2024-08-31T08:55:28.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2024-08-31T08:55:28.328 INFO:tasks.rbd_mirror_thrash:waiting for 102 secs before thrashing 2024-08-31T08:55:28.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9500 IOs, 3907014656 bytes 2024-08-31T08:55:28.662 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9600 IOs, 3983721472 bytes 2024-08-31T08:55:28.733 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9700 IOs, 3997308928 bytes 2024-08-31T08:55:28.853 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9800 IOs, 4045169664 bytes 2024-08-31T08:55:28.932 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:9900 IOs, 4055918592 bytes 2024-08-31T08:55:29.083 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10000 IOs, 4114216960 bytes 2024-08-31T08:55:29.194 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10100 IOs, 4154859520 bytes 2024-08-31T08:55:29.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10200 IOs, 4172220416 bytes 2024-08-31T08:55:29.384 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~292s 2024-08-31T08:55:29.409 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10300 IOs, 4194027520 bytes 2024-08-31T08:55:29.529 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10400 IOs, 4235293696 bytes 2024-08-31T08:55:29.657 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10500 IOs, 4272027648 bytes 2024-08-31T08:55:29.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:10600 IOs, 4348316672 bytes 2024-08-31T08:55:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ error_code=124 2024-08-31T08:55:29.896 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ set -e 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ '[' 124 -eq 124 ']' 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local state=present 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id= 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local s 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n '' 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_10 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:55:29.897 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:55:29.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_10 2024-08-31T08:55:29.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-08-31T08:55:29.898 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:55:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ image_id=178bf5a89c79 2024-08-31T08:55:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2024-08-31T08:55:29.941 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ sleep 0.1 2024-08-31T08:55:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test_image_present cluster1 mirror image_10 present 178bf5a89c79 2024-08-31T08:55:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster1 2024-08-31T08:55:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:30.042 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local test_state=present 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image_id=178bf5a89c79 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_state=deleted 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local current_image_id 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ get_image_id cluster1 mirror image_10 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster=cluster1 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local pool=mirror 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local image=image_10 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2024-08-31T08:55:30.043 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2024-08-31T08:55:30.087 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_image_id=178bf5a89c79 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -n 178bf5a89c79 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test -z 178bf5a89c79 -o 178bf5a89c79 = 178bf5a89c79 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ current_state=present 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ test present = present 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ return 0 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ disable_mirror cluster2 mirror image_10 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local cluster=cluster2 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local pool=mirror 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ local image=image_10 2024-08-31T08:55:30.088 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2024-08-31T08:55:34.789 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~297s 2024-08-31T08:55:40.193 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~303s 2024-08-31T08:55:40.193 INFO:tasks.daemonwatchdog.daemon_watchdog:BARK! unmounting mounts and killing all daemons 2024-08-31T08:55:40.596 INFO:tasks.ceph.cluster2.osd.1:Sent signal 15 2024-08-31T08:55:40.596 INFO:tasks.ceph.cluster2.osd.2:Sent signal 15 2024-08-31T08:55:40.597 INFO:tasks.ceph.cluster2.mon.a:Sent signal 15 2024-08-31T08:55:40.597 INFO:tasks.ceph.cluster2.mgr.x:Sent signal 15 2024-08-31T08:55:40.597 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f5b08f78640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 (PID: 21979) UID: 0 2024-08-31T08:55:40.597 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f5b08f78640 -1 osd.1 502 *** Got signal Terminated *** 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.osd.1.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f5b08f78640 -1 osd.1 502 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f6cbd956640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 (PID: 21980) UID: 0 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f6cbd956640 -1 osd.2 502 *** Got signal Terminated *** 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.osd.2.smithi167.stderr:2024-08-31T08:55:40.596+0000 7f6cbd956640 -1 osd.2 502 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.mon.a.smithi167.stderr:2024-08-31T08:55:40.597+0000 7f5b504be640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mon -f --cluster cluster2 -i a (PID: 21671) UID: 0 2024-08-31T08:55:40.598 INFO:tasks.ceph.cluster2.mon.a.smithi167.stderr:2024-08-31T08:55:40.597+0000 7f5b504be640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-31T08:55:40.599 INFO:tasks.ceph.cluster2.mgr.x.smithi012.stderr:2024-08-31T08:55:40.597+0000 7ffad0760640 -1 received signal: Terminated from /usr/bin/python3 /bin/daemon-helper kill ceph-mgr -f --cluster cluster2 -i x (PID: 26558) UID: 0 2024-08-31T08:55:40.599 INFO:tasks.ceph.cluster2.mgr.x.smithi012.stderr:2024-08-31T08:55:40.597+0000 7ffad0760640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-08-31T08:56:27.398 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2024-08-31T08:56:27.399 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2024-08-31T08:56:27.399 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-08-31T08:56:27.400 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T08:56:27.399+0000 7f388ab45640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 44374) UID: 1000 2024-08-31T08:56:29.400 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2024-08-31T08:56:29.400 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T08:57:10.329 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9700099723859501) > weight (0.25) 2024-08-31T08:57:10.329 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9138189939531021) > weight (0.25) 2024-08-31T08:57:10.329 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2024-08-31T08:57:10.330 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2024-08-31T08:57:10.330 INFO:tasks.rbd_mirror_thrash:waiting for 2 secs before reviving daemons 2024-08-31T08:57:10.331 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T08:57:10.330+0000 7f71fca64640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 48572) UID: 1000 2024-08-31T08:57:12.331 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2024-08-31T08:57:12.331 INFO:teuthology.orchestra.run:waiting for 600 2024-08-31T09:06:33.542 INFO:tasks.rbd_mirror_thrash:Failed to stop cluster1.client.mirror.0 2024-08-31T09:06:33.543 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 6 2024-08-31T09:06:33.543 ERROR:tasks.rbd_mirror_thrash:exception: Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/rbd_mirror_thrash.py", line 84, in _run self.do_thrash() File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/rbd_mirror_thrash.py", line 156, in do_thrash run.wait([daemon.proc], timeout=600) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 473, in wait check_time() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/contextutil.py", line 134, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: reached maximum tries (100) after waiting for 600 seconds 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:*** Caught signal (Aborted) ** 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: in thread 7f388dba2640 thread_name:rbd-mirror 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f388e03e6f0] 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f388e0867ea] 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f388e4d56b0] 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 5: rbd-mirror(+0x3cf81b) [0x55ee6ead081b] 2024-08-31T09:06:33.545 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 6: (rbd::mirror::PoolReplayer::shut_down()+0x103) [0x55ee6eae1493] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 7: (rbd::mirror::PoolReplayer::~PoolReplayer()+0x10) [0x55ee6ead42b0] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 8: rbd-mirror(+0xa62fb8) [0x55ee6f163fb8] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 9: rbd-mirror(+0xa62fa0) [0x55ee6f163fa0] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 10: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55ee6ead253a] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 11: main() 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 12: /lib64/libc.so.6(+0x29590) [0x7f388e029590] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 13: __libc_start_main() 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 14: _start() 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:2024-08-31T09:06:33.543+0000 7f388dba2640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: in thread 7f388dba2640 thread_name:rbd-mirror 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f388e03e6f0] 2024-08-31T09:06:33.546 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f388e0867ea] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f388e4d56b0] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 5: rbd-mirror(+0x3cf81b) [0x55ee6ead081b] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 6: (rbd::mirror::PoolReplayer::shut_down()+0x103) [0x55ee6eae1493] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 7: (rbd::mirror::PoolReplayer::~PoolReplayer()+0x10) [0x55ee6ead42b0] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 8: rbd-mirror(+0xa62fb8) [0x55ee6f163fb8] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 9: rbd-mirror(+0xa62fa0) [0x55ee6f163fa0] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 10: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55ee6ead253a] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 11: main() 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 12: /lib64/libc.so.6(+0x29590) [0x7f388e029590] 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 13: __libc_start_main() 2024-08-31T09:06:33.547 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 14: _start() 2024-08-31T09:06:33.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:06:33.548 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.556 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: -2759> 2024-08-31T08:56:27.399+0000 7f388ab45640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 44374) UID: 1000 2024-08-31T09:06:33.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 0> 2024-08-31T09:06:33.543+0000 7f388dba2640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:06:33.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: in thread 7f388dba2640 thread_name:rbd-mirror 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f388e03e6f0] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f388e0867ea] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f388e4d56b0] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 5: rbd-mirror(+0x3cf81b) [0x55ee6ead081b] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 6: (rbd::mirror::PoolReplayer::shut_down()+0x103) [0x55ee6eae1493] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 7: (rbd::mirror::PoolReplayer::~PoolReplayer()+0x10) [0x55ee6ead42b0] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 8: rbd-mirror(+0xa62fb8) [0x55ee6f163fb8] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 9: rbd-mirror(+0xa62fa0) [0x55ee6f163fa0] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 10: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55ee6ead253a] 2024-08-31T09:06:33.559 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 11: main() 2024-08-31T09:06:33.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 12: /lib64/libc.so.6(+0x29590) [0x7f388e029590] 2024-08-31T09:06:33.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 13: __libc_start_main() 2024-08-31T09:06:33.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 14: _start() 2024-08-31T09:06:33.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:06:33.560 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: -9999> 2024-08-31T08:56:27.399+0000 7f388ab45640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 44374) UID: 1000 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: -9998> 2024-08-31T09:06:33.543+0000 7f388dba2640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: in thread 7f388dba2640 thread_name:rbd-mirror 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f388e03e6f0] 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f388e0867ea] 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f388e4d56b0] 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 5: rbd-mirror(+0x3cf81b) [0x55ee6ead081b] 2024-08-31T09:06:33.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 6: (rbd::mirror::PoolReplayer::shut_down()+0x103) [0x55ee6eae1493] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 7: (rbd::mirror::PoolReplayer::~PoolReplayer()+0x10) [0x55ee6ead42b0] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 8: rbd-mirror(+0xa62fb8) [0x55ee6f163fb8] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 9: rbd-mirror(+0xa62fa0) [0x55ee6f163fa0] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 10: (rbd::mirror::Mirror::~Mirror()+0xca) [0x55ee6ead253a] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 11: main() 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 12: /lib64/libc.so.6(+0x29590) [0x7f388e029590] 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 13: __libc_start_main() 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 14: _start() 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:06:33.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr: 2024-08-31T09:06:33.944 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.smithi167.stderr:daemon-helper: command crashed with signal 6 2024-08-31T09:06:35.335 INFO:tasks.daemonwatchdog.daemon_watchdog:thrasher.rbd_mirror.[cluster1] failed 2024-08-31T09:06:35.335 ERROR:tasks.daemonwatchdog.daemon_watchdog:exception: Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/daemonwatchdog.py", line 42, in _run self.watch() File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/daemonwatchdog.py", line 120, in watch thrasher.stop_and_join() AttributeError: 'RBDMirrorThrasher' object has no attribute 'stop_and_join' 2024-08-31T09:07:16.472 INFO:tasks.rbd_mirror_thrash:Failed to stop cluster2.client.mirror.2 2024-08-31T09:07:16.473 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 6 2024-08-31T09:07:16.473 ERROR:tasks.rbd_mirror_thrash:exception: Traceback (most recent call last): File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/rbd_mirror_thrash.py", line 84, in _run self.do_thrash() File "/home/teuthworker/src/github.com_ceph_ceph-c_dc9f1ccbfcab74acda17f39d6ab018266071e760/qa/tasks/rbd_mirror_thrash.py", line 156, in do_thrash run.wait([daemon.proc], timeout=600) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 473, in wait check_time() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/contextutil.py", line 134, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: reached maximum tries (100) after waiting for 600 seconds 2024-08-31T09:07:16.474 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:*** Caught signal (Aborted) ** 2024-08-31T09:07:16.474 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: in thread 7f71ffac1640 thread_name:rbd-mirror 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f720003e6f0] 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f72000867ea] 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f72004d56b0] 2024-08-31T09:07:16.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 5: /lib64/librados.so.2(+0x14a11b) [0x7f7201d1e11b] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 6: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map >, std::less, std::allocator > > > >*)+0x38) [0x5624ef7a9b88] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 7: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x5624ef7aba08] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 8: (rbd::mirror::Mirror::run()+0xb4) [0x5624ef78aa34] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 9: main() 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 10: /lib64/libc.so.6(+0x29590) [0x7f7200029590] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 11: __libc_start_main() 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 12: _start() 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:2024-08-31T09:07:16.474+0000 7f71ffac1640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: in thread 7f71ffac1640 thread_name:rbd-mirror 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f720003e6f0] 2024-08-31T09:07:16.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f72000867ea] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f72004d56b0] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 5: /lib64/librados.so.2(+0x14a11b) [0x7f7201d1e11b] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 6: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map >, std::less, std::allocator > > > >*)+0x38) [0x5624ef7a9b88] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 7: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x5624ef7aba08] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 8: (rbd::mirror::Mirror::run()+0xb4) [0x5624ef78aa34] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 9: main() 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 10: /lib64/libc.so.6(+0x29590) [0x7f7200029590] 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 11: __libc_start_main() 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 12: _start() 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:07:16.477 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9558> 2024-08-31T08:48:53.158+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3734b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9555> 2024-08-31T08:48:53.158+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/b080f365-a5bd-4cee-9717-8d5cc278ced7] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9407> 2024-08-31T08:48:56.046+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3877520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9404> 2024-08-31T08:48:56.046+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4109680 [3/1fd95124-6f6d-4ddf-a6ce-7b0aec68c31f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -8970> 2024-08-31T08:49:22.404+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3732680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.481 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -8967> 2024-08-31T08:49:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/4dc58479-be23-4acf-8087-3091c1bbb4e9] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -8143> 2024-08-31T08:50:07.489+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a949c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -8140> 2024-08-31T08:50:07.489+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -7974> 2024-08-31T08:50:22.404+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a94680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -7971> 2024-08-31T08:50:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -7447> 2024-08-31T08:50:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5c820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.482 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -7444> 2024-08-31T08:50:52.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/c3ddec03-4fa7-402e-9daf-c0d180f4d77c] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.483 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -6826> 2024-08-31T08:51:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.483 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -6823> 2024-08-31T08:51:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/6f56ac0c-13df-4803-a7da-5f99d1121a6f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -5848> 2024-08-31T08:52:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f65040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -5845> 2024-08-31T08:52:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/1464d851-4439-47fa-803c-e0839c0e51d1] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -5224> 2024-08-31T08:52:52.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -5221> 2024-08-31T08:52:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.484 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -4989> 2024-08-31T08:53:03.487+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f62680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -4986> 2024-08-31T08:53:03.487+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -4596> 2024-08-31T08:53:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3735040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.485 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -4593> 2024-08-31T08:53:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/ea665289-b2d3-4e6c-b5fa-e7e081983ada] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -3623> 2024-08-31T08:54:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -3620> 2024-08-31T08:54:22.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/900e5810-e68f-4926-b489-2be949a4a59b] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.486 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -2979> 2024-08-31T08:54:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5b380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -2976> 2024-08-31T08:54:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/d109198c-6282-42c0-8b36-959ff5918e18] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.487 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -1663> 2024-08-31T08:57:10.330+0000 7f71fca64640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 48572) UID: 1000 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 0> 2024-08-31T09:07:16.474+0000 7f71ffac1640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: in thread 7f71ffac1640 thread_name:rbd-mirror 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f720003e6f0] 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f72000867ea] 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f72004d56b0] 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 5: /lib64/librados.so.2(+0x14a11b) [0x7f7201d1e11b] 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 6: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map >, std::less, std::allocator > > > >*)+0x38) [0x5624ef7a9b88] 2024-08-31T09:07:16.489 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 7: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x5624ef7aba08] 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 8: (rbd::mirror::Mirror::run()+0xb4) [0x5624ef78aa34] 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 9: main() 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 10: /lib64/libc.so.6(+0x29590) [0x7f7200029590] 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 11: __libc_start_main() 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 12: _start() 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:07:16.490 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9999> 2024-08-31T08:48:53.158+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3734b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9998> 2024-08-31T08:48:53.158+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/b080f365-a5bd-4cee-9717-8d5cc278ced7] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9997> 2024-08-31T08:48:56.046+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3877520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9996> 2024-08-31T08:48:56.046+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4109680 [3/1fd95124-6f6d-4ddf-a6ce-7b0aec68c31f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9995> 2024-08-31T08:49:22.404+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3732680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9994> 2024-08-31T08:49:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/4dc58479-be23-4acf-8087-3091c1bbb4e9] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9993> 2024-08-31T08:50:07.489+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a949c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9992> 2024-08-31T08:50:07.489+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9991> 2024-08-31T08:50:22.404+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3a94680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9990> 2024-08-31T08:50:22.404+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/0ea29e1a-0fcf-437a-b970-4c0bf0b4b7fc] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9989> 2024-08-31T08:50:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5c820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.492 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9988> 2024-08-31T08:50:52.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/c3ddec03-4fa7-402e-9daf-c0d180f4d77c] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9987> 2024-08-31T08:51:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9986> 2024-08-31T08:51:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/6f56ac0c-13df-4803-a7da-5f99d1121a6f] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9985> 2024-08-31T08:52:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f65040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9984> 2024-08-31T08:52:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/1464d851-4439-47fa-803c-e0839c0e51d1] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9983> 2024-08-31T08:52:52.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f64ea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9982> 2024-08-31T08:52:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9981> 2024-08-31T08:53:03.487+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f62680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9980> 2024-08-31T08:53:03.487+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/c8727ab2-068f-47b6-bdb9-43ae37872a41] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9979> 2024-08-31T08:53:22.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3735040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9978> 2024-08-31T08:53:22.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f3676280 [3/ea665289-b2d3-4e6c-b5fa-e7e081983ada] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9977> 2024-08-31T08:54:22.405+0000 7f71f5255640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5d040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.493 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9976> 2024-08-31T08:54:22.405+0000 7f71fc263640 -1 rbd::mirror::ImageReplayer: 0x5624f41e7680 [3/900e5810-e68f-4926-b489-2be949a4a59b] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9975> 2024-08-31T08:54:52.405+0000 7f71f5a56640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x5624f3f5b380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9974> 2024-08-31T08:54:52.405+0000 7f71fba62640 -1 rbd::mirror::ImageReplayer: 0x5624f4419900 [3/d109198c-6282-42c0-8b36-959ff5918e18] operator(): start failed: (85) Interrupted system call should be restarted 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9973> 2024-08-31T08:57:10.330+0000 7f71fca64640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 48572) UID: 1000 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: -9972> 2024-08-31T09:07:16.474+0000 7f71ffac1640 -1 *** Caught signal (Aborted) ** 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: in thread 7f71ffac1640 thread_name:rbd-mirror 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev) 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 1: /lib64/libc.so.6(+0x3e6f0) [0x7f720003e6f0] 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2: /lib64/libc.so.6(+0x867ea) [0x7f72000867ea] 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 3: pthread_cond_wait() 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 4: (std::condition_variable::wait(std::unique_lock&)+0x10) [0x7f72004d56b0] 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 5: /lib64/librados.so.2(+0x14a11b) [0x7f7201d1e11b] 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 6: (rbd::mirror::ClusterWatcher::read_pool_peers(std::map >, std::less, std::allocator > > > >*)+0x38) [0x5624ef7a9b88] 2024-08-31T09:07:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 7: (rbd::mirror::ClusterWatcher::refresh_pools()+0x78) [0x5624ef7aba08] 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 8: (rbd::mirror::Mirror::run()+0xb4) [0x5624ef78aa34] 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 9: main() 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 10: /lib64/libc.so.6(+0x29590) [0x7f7200029590] 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 11: __libc_start_main() 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 12: _start() 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2024-08-31T09:07:16.495 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr: 2024-08-31T09:07:16.875 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.smithi167.stderr:daemon-helper: command crashed with signal 6 2024-08-31T14:20:40.959 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:Terminated 2024-08-31T14:20:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ cleanup 143 2024-08-31T14:20:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local error_code=143 2024-08-31T14:20:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ set +e 2024-08-31T14:20:41.211 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ '[' 143 -ne 0 ']' 2024-08-31T14:20:41.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ status 2024-08-31T14:20:41.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ local cluster daemon image_pool image_ns image 2024-08-31T14:20:41.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for cluster in ${CLUSTER1} ${CLUSTER2} 2024-08-31T14:20:41.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 status' 2024-08-31T14:20:41.212 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 status 2024-08-31T14:20:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:41.213 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster1 -s 2024-08-31T14:20:41.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: cluster: 2024-08-31T14:20:41.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: c26ce5cc-1226-40b9-9482-3cb99b32adb2 2024-08-31T14:20:41.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: HEALTH_OK 2024-08-31T14:20:41.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:41.470 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: services: 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mon: 1 daemons, quorum a (age 6h) 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mgr: x(active, since 6h) 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: osd: 3 osds: 3 up (since 6h), 3 in (since 6h) 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: rbd-mirror: 3 daemons active (1 hosts) 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: data: 2024-08-31T14:20:41.471 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: pools: 4 pools, 137 pgs 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: objects: 39 objects, 577 KiB 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: usage: 473 MiB used, 270 GiB / 270 GiB avail 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: pgs: 137 active+clean 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: io: 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client: 1.9 KiB/s rd, 341 B/s wr, 1 op/s rd, 0 op/s wr 2024-08-31T14:20:41.472 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:41.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:41.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster1 service dump 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:{ 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "epoch": 129, 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "modified": "2024-08-31T14:15:07.092352+0000", 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "services": { 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "rbd-mirror": { 2024-08-31T14:20:41.845 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "daemons": { 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "summary": "", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "5257": { 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_epoch": 23, 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_stamp": "2024-08-31T08:43:28.449725+0000", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "gid": 5257, 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "addr": "172.21.15.167:0/2894293119", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "metadata": { 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "arch": "x86_64", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_release": "squid", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version": "ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev)", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version_short": "19.3.0-4620-gdc9f1ccb", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "cpu": "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz", 2024-08-31T14:20:41.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro": "centos", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_description": "CentOS Stream 9", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_version": "9", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "hostname": "smithi167", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "id": "mirror.3", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "instance_id": "5257", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_description": "#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_version": "5.14.0-503.el9.x86_64", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_swap_kb": "0", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_total_kb": "32487372", 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "os": "Linux" 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "task_status": {} 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:41.847 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "6024": { 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_epoch": 31, 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_stamp": "2024-08-31T08:50:09.467871+0000", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "gid": 6024, 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "addr": "172.21.15.167:0/485063082", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "metadata": { 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "arch": "x86_64", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_release": "squid", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version": "ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev)", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version_short": "19.3.0-4620-gdc9f1ccb", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "cpu": "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro": "centos", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_description": "CentOS Stream 9", 2024-08-31T14:20:41.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_version": "9", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "hostname": "smithi167", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "id": "mirror.2", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "instance_id": "6024", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_description": "#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_version": "5.14.0-503.el9.x86_64", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_swap_kb": "0", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_total_kb": "32487372", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "os": "Linux" 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "task_status": {} 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "6093": { 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_epoch": 33, 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "start_stamp": "2024-08-31T08:53:06.482576+0000", 2024-08-31T14:20:41.849 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "gid": 6093, 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "addr": "172.21.15.167:0/2085915062", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "metadata": { 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "arch": "x86_64", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_release": "squid", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version": "ceph version 19.3.0-4620-gdc9f1ccb (dc9f1ccbfcab74acda17f39d6ab018266071e760) squid (dev)", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "ceph_version_short": "19.3.0-4620-gdc9f1ccb", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "cpu": "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro": "centos", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_description": "CentOS Stream 9", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "distro_version": "9", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "hostname": "smithi167", 2024-08-31T14:20:41.850 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "id": "mirror.1", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "instance_id": "6093", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_description": "#1 SMP PREEMPT_DYNAMIC Thu Aug 22 17:03:23 UTC 2024", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "kernel_version": "5.14.0-503.el9.x86_64", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_swap_kb": "0", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "mem_total_kb": "32487372", 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "os": "Linux" 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "task_status": {} 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:41.851 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:41.852 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:} 2024-08-31T14:20:41.855 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:41.856 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster1 service status 2024-08-31T14:20:42.214 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:{ 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "rbd-mirror": { 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "5257": { 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status_stamp": "2024-08-31T14:20:33.634562+0000", 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "last_beacon": "2024-08-31T14:20:38.634605+0000", 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status": { 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "json": "{\"3\":{\"name\":\"mirror\",\"callouts\":{},\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0,\"instance_id\":\"5260\",\"namespaces\":{\"ns1\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0},\"ns2\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0}}},\"4\":{\"name\":\"mirror_parent\",\"callouts\":{},\"image_assigned_count\":0,\"image_error_count\":0,\"image_local_count\":0,\"image_remote_count\":0,\"image_warning_count\":0,\"instance_id\":\"5263\",\"leader\":true}}" 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "6024": { 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status_stamp": "2024-08-31T14:20:14.605778+0000", 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "last_beacon": "2024-08-31T14:20:39.606915+0000", 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status": { 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "json": "{\"3\":{\"name\":\"mirror\",\"callouts\":{},\"image_assigned_count\":1,\"image_error_count\":0,\"image_local_count\":1,\"image_remote_count\":1,\"image_warning_count\":0,\"instance_id\":\"6027\",\"leader\":true,\"namespaces\":{\"ns1\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_local_count\":0,\"image_remote_count\":0,\"image_warning_count\":0},\"ns2\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_local_count\":0,\"image_remote_count\":0,\"image_warning_count\":0}}},\"4\":{\"name\":\"mirror_parent\",\"callouts\":{},\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0,\"instance_id\":\"6030\"}}" 2024-08-31T14:20:42.215 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: }, 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "6093": { 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status_stamp": "2024-08-31T14:20:41.679081+0000", 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "last_beacon": "2024-08-31T14:20:41.679081+0000", 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "status": { 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: "json": "{\"3\":{\"name\":\"mirror\",\"callouts\":{},\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0,\"instance_id\":\"6096\",\"namespaces\":{\"ns1\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0},\"ns2\":{\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0}}},\"4\":{\"name\":\"mirror_parent\",\"callouts\":{},\"image_assigned_count\":0,\"image_error_count\":0,\"image_warning_count\":0,\"instance_id\":\"6099\"}}" 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: } 2024-08-31T14:20:42.216 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:} 2024-08-31T14:20:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.226 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_pool in ${POOL} ${PARENT_POOL} 2024-08-31T14:20:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror images 2024-08-31T14:20:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror images' 2024-08-31T14:20:42.227 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' ls -l 2024-08-31T14:20:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:NAME SIZE PARENT FMT PROT LOCK 2024-08-31T14:20:42.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image_10 512 MiB 2 excl 2024-08-31T14:20:42.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror mirror pool info 2024-08-31T14:20:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror mirror pool info' 2024-08-31T14:20:42.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' mirror pool info 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mode: image 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Site Name: cluster1 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Peer Sites: 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:UUID: e82eadfc-e2b2-418a-998f-413f4a86c94d 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Name: cluster2 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirror UUID: b2ed8de6-469c-4293-9d02-39498e569bf2 2024-08-31T14:20:42.312 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Direction: rx-tx 2024-08-31T14:20:42.313 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Client: client.mirror 2024-08-31T14:20:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.314 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror mirror pool status 2024-08-31T14:20:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror mirror pool status' 2024-08-31T14:20:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:42.315 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' mirror pool status --verbose 2024-08-31T14:20:42.351 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:health: WARNING 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:daemon health: OK 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image health: WARNING 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:images: 1 total 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 1 unknown 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:DAEMONS 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 5257: 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 5260 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.3 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.352 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6024: 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6027 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.2 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: true 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6093: 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6096 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.1 2024-08-31T14:20:42.353 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.354 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:IMAGES 2024-08-31T14:20:42.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image_10: 2024-08-31T14:20:42.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: global_id: 48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T14:20:42.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: up+replaying 2024-08-31T14:20:42.369 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1725094516,"remote_snapshot_timestamp":1725094516,"replay_state":"idle"} 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: service: mirror.2 on smithi167 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: last_update: 2024-08-31 14:20:39 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: peer_sites: 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: name: cluster2 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: state: down+starting_replay 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: description: starting replay 2024-08-31T14:20:42.370 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: last_update: 2024-08-31 09:06:52 2024-08-31T14:20:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.372 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror --namespace '' ls 2024-08-31T14:20:42.410 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image in `rbd --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" ls 2>/dev/null` 2024-08-31T14:20:42.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image image_10 info 2024-08-31T14:20:42.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'image image_10 info' 2024-08-31T14:20:42.411 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' info image_10 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:rbd image 'image_10': 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: size 512 MiB in 128 objects 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: order 22 (4 MiB objects) 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: snapshot_count: 1 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: id: 178bf5a89c79 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: block_name_prefix: rbd_data.178bf5a89c79 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: format: 2 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: op_features: 2024-08-31T14:20:42.454 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: flags: 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: create_timestamp: Sat Aug 31 08:55:17 2024 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: access_timestamp: Sat Aug 31 08:55:17 2024 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: modify_timestamp: Sat Aug 31 08:55:17 2024 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring state: enabled 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring mode: snapshot 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring global id: 48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T14:20:42.455 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: mirroring primary: false 2024-08-31T14:20:42.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image image_10 journal status 2024-08-31T14:20:42.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'image image_10 journal status' 2024-08-31T14:20:42.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' journal status --image image_10 2024-08-31T14:20:42.501 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: journaling is not enabled for image image_10 2024-08-31T14:20:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.504 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image image_10 snapshots 2024-08-31T14:20:42.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'image image_10 snapshots' 2024-08-31T14:20:42.505 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace '' snap ls --all image_10 2024-08-31T14:20:42.548 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:SNAPID NAME SIZE PROTECTED TIMESTAMP NAMESPACE 2024-08-31T14:20:42.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 195 .mirror.non_primary.48e1a4e8-585b-4ff0-b874-bfa84c0c2855.402f1f2a-b274-490a-9622-2946e045b731 512 MiB Sat Aug 31 08:55:18 2024 mirror (non-primary peer_uuids:[] b2ed8de6-469c-4293-9d02-39498e569bf2:194 copied) 2024-08-31T14:20:42.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror rbd_mirroring omap vals' 2024-08-31T14:20:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror rbd_mirroring omap vals 2024-08-31T14:20:42.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace '' listomapvals rbd_mirroring 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:global_48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (16 bytes) : 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 0c 00 00 00 31 37 38 62 66 35 61 38 39 63 37 39 |....178bf5a89c79| 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image_178bf5a89c79 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (48 bytes) : 2024-08-31T14:20:42.788 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 02 01 2a 00 00 00 24 00 00 00 34 38 65 31 61 34 |..*...$...48e1a4| 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 65 38 2d 35 38 35 62 2d 34 66 66 30 2d 62 38 37 |e8-585b-4ff0-b87| 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 34 2d 62 66 61 38 34 63 30 63 32 38 35 35 01 01 |4-bfa84c0c2855..| 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000030 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_mode 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (4 bytes) : 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 00 00 00 |....| 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000004 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_peer_e82eadfc-e2b2-418a-998f-413f4a86c94d 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (132 bytes) : 2024-08-31T14:20:42.789 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 02 01 7e 00 00 00 24 00 00 00 65 38 32 65 61 64 |..~...$...e82ead| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 66 63 2d 65 32 62 32 2d 34 31 38 61 2d 39 39 38 |fc-e2b2-418a-998| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 66 2d 34 31 33 66 34 61 38 36 63 39 34 64 08 00 |f-413f4a86c94d..| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000030 00 00 63 6c 75 73 74 65 72 32 0d 00 00 00 63 6c |..cluster2....cl| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000040 69 65 6e 74 2e 6d 69 72 72 6f 72 ff ff ff ff ff |ient.mirror.....| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000050 ff ff ff 02 24 00 00 00 62 32 65 64 38 64 65 36 |....$...b2ed8de6| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000060 2d 34 36 39 63 2d 34 32 39 33 2d 39 64 30 32 2d |-469c-4293-9d02-| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000070 33 39 34 39 38 65 35 36 39 62 66 32 b7 26 d3 66 |39498e569bf2.&.f| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000080 57 80 37 00 |W.7.| 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000084 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_uuid 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (36 bytes) : 2024-08-31T14:20:42.790 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 36 62 31 30 35 36 36 32 2d 63 62 38 37 2d 34 31 |6b105662-cb87-41| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 33 36 2d 62 65 34 37 2d 66 32 61 62 34 36 65 35 |36-be47-f2ab46e5| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 62 31 65 36 |b1e6| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000024 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:remote_status_global_48e1a4e8-585b-4ff0-b874-bfa84c0c2855_b2ed8de6-469c-4293-9d02-39498e569bf2 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (85 bytes) : 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 01 2c 00 00 00 08 06 15 00 00 00 00 00 00 01 |..,.............| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 01 01 1c 00 00 00 03 00 00 00 b1 72 72 97 10 00 |...........rr...| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 00 00 02 00 00 00 ac 15 0f a7 00 00 00 00 00 00 |................| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000030 00 00 01 01 1d 00 00 00 03 0f 00 00 00 73 74 61 |.............sta| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000040 72 74 69 6e 67 20 72 65 70 6c 61 79 2c dd d2 66 |rting replay,..f| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000050 c5 71 e4 17 00 |.q...| 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000055 2024-08-31T14:20:42.791 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:status_global_48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (284 bytes) : 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 01 2c 00 00 00 08 8b 17 00 00 00 00 00 00 01 |..,.............| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 01 01 1c 00 00 00 03 00 00 00 ce be 14 e4 10 00 |................| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 00 00 02 00 00 00 ac 15 0f a7 00 00 00 00 00 00 |................| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000030 00 00 01 01 e4 00 00 00 04 d6 00 00 00 72 65 70 |.............rep| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000040 6c 61 79 69 6e 67 2c 20 7b 22 62 79 74 65 73 5f |laying, {"bytes_| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000050 70 65 72 5f 73 65 63 6f 6e 64 22 3a 30 2e 30 2c |per_second":0.0,| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000060 22 62 79 74 65 73 5f 70 65 72 5f 73 6e 61 70 73 |"bytes_per_snaps| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000070 68 6f 74 22 3a 30 2e 30 2c 22 6c 61 73 74 5f 73 |hot":0.0,"last_s| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000080 6e 61 70 73 68 6f 74 5f 62 79 74 65 73 22 3a 30 |napshot_bytes":0| 2024-08-31T14:20:42.792 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000090 2c 22 6c 61 73 74 5f 73 6e 61 70 73 68 6f 74 5f |,"last_snapshot_| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000a0 73 79 6e 63 5f 73 65 63 6f 6e 64 73 22 3a 30 2c |sync_seconds":0,| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000b0 22 6c 6f 63 61 6c 5f 73 6e 61 70 73 68 6f 74 5f |"local_snapshot_| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000c0 74 69 6d 65 73 74 61 6d 70 22 3a 31 37 32 35 30 |timestamp":17250| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000d0 39 34 35 31 36 2c 22 72 65 6d 6f 74 65 5f 73 6e |94516,"remote_sn| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000e0 61 70 73 68 6f 74 5f 74 69 6d 65 73 74 61 6d 70 |apshot_timestamp| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:000000f0 22 3a 31 37 32 35 30 39 34 35 31 36 2c 22 72 65 |":1725094516,"re| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000100 70 6c 61 79 5f 73 74 61 74 65 22 3a 22 69 64 6c |play_state":"idl| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000110 65 22 7d b7 26 d3 66 3b 92 fb 22 00 |e"}.&.f;..".| 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:0000011c 2024-08-31T14:20:42.793 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror rbd_mirror_leader omap vals 2024-08-31T14:20:42.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror rbd_mirror_leader omap vals' 2024-08-31T14:20:42.794 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace '' listomapvals rbd_mirror_leader 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image_map_48e1a4e8-585b-4ff0-b874-bfa84c0c2855 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (36 bytes) : 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 01 1e 00 00 00 04 00 00 00 36 30 32 37 74 da |..........6027t.| 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 d2 66 81 d2 a9 33 0a 00 00 00 01 01 04 00 00 00 |.f...3..........| 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 00 00 00 00 |....| 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000024 2024-08-31T14:20:42.825 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_5260 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6027 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6096 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:42.826 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.827 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:42.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns1 images 2024-08-31T14:20:42.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns1 images' 2024-08-31T14:20:42.828 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 ls -l 2024-08-31T14:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.869 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns1 mirror pool info 2024-08-31T14:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns1 mirror pool info' 2024-08-31T14:20:42.870 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 mirror pool info 2024-08-31T14:20:42.902 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mode: image 2024-08-31T14:20:42.904 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns1 mirror pool status 2024-08-31T14:20:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns1 mirror pool status' 2024-08-31T14:20:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:42.905 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns1 mirror pool status --verbose 2024-08-31T14:20:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:health: OK 2024-08-31T14:20:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:daemon health: OK 2024-08-31T14:20:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image health: OK 2024-08-31T14:20:42.942 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:images: 0 total 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:DAEMONS 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 5257: 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 5260 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.3 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6024: 2024-08-31T14:20:42.943 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6027 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.2 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: true 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6093: 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6096 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.1 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:42.944 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.945 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:IMAGES 2024-08-31T14:20:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:42.947 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror --namespace ns1 ls 2024-08-31T14:20:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns1 rbd_mirroring omap vals 2024-08-31T14:20:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns1 rbd_mirroring omap vals' 2024-08-31T14:20:42.985 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace ns1 listomapvals rbd_mirroring 2024-08-31T14:20:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_mode 2024-08-31T14:20:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (4 bytes) : 2024-08-31T14:20:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 00 00 00 |....| 2024-08-31T14:20:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000004 2024-08-31T14:20:43.019 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_uuid 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (36 bytes) : 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 37 65 36 63 64 64 61 36 2d 39 31 35 63 2d 34 36 |7e6cdda6-915c-46| 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 38 36 2d 38 30 34 63 2d 31 31 61 64 30 36 34 65 |86-804c-11ad064e| 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 64 37 33 65 |d73e| 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000024 2024-08-31T14:20:43.020 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns1 rbd_mirror_leader omap vals' 2024-08-31T14:20:43.021 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns1 rbd_mirror_leader omap vals 2024-08-31T14:20:43.022 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace ns1 listomapvals rbd_mirror_leader 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_5260 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6027 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.055 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6096 2024-08-31T14:20:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.056 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.057 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns2 images' 2024-08-31T14:20:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns2 images 2024-08-31T14:20:43.058 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 ls -l 2024-08-31T14:20:43.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.095 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns2 mirror pool info 2024-08-31T14:20:43.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns2 mirror pool info' 2024-08-31T14:20:43.096 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 mirror pool info 2024-08-31T14:20:43.132 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mode: image 2024-08-31T14:20:43.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.134 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns2 mirror pool status 2024-08-31T14:20:43.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns2 mirror pool status' 2024-08-31T14:20:43.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:43.135 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror --namespace ns2 mirror pool status --verbose 2024-08-31T14:20:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:health: OK 2024-08-31T14:20:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:daemon health: OK 2024-08-31T14:20:43.173 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image health: OK 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:images: 0 total 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:DAEMONS 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 5257: 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 5260 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.3 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.174 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6024: 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6027 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.2 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: true 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.175 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6093: 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6096 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.1 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.176 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:IMAGES 2024-08-31T14:20:43.178 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.179 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror --namespace ns2 ls 2024-08-31T14:20:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns2 rbd_mirroring omap vals 2024-08-31T14:20:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns2 rbd_mirroring omap vals' 2024-08-31T14:20:43.217 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace ns2 listomapvals rbd_mirroring 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_mode 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (4 bytes) : 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 00 00 00 |....| 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000004 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_uuid 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (36 bytes) : 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 61 61 63 31 37 65 32 35 2d 63 30 32 66 2d 34 32 |aac17e25-c02f-42| 2024-08-31T14:20:43.251 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 33 61 2d 61 61 34 38 2d 64 33 30 30 61 37 36 35 |3a-aa48-d300a765| 2024-08-31T14:20:43.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 36 61 63 32 |6ac2| 2024-08-31T14:20:43.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000024 2024-08-31T14:20:43.252 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror ns2 rbd_mirror_leader omap vals' 2024-08-31T14:20:43.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror ns2 rbd_mirror_leader omap vals 2024-08-31T14:20:43.253 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror --namespace ns2 listomapvals rbd_mirror_leader 2024-08-31T14:20:43.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_5260 2024-08-31T14:20:43.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.287 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6027 2024-08-31T14:20:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6096 2024-08-31T14:20:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.288 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.289 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_pool in ${POOL} ${PARENT_POOL} 2024-08-31T14:20:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent images 2024-08-31T14:20:43.290 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent images' 2024-08-31T14:20:43.291 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' ls -l 2024-08-31T14:20:43.325 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent mirror pool info 2024-08-31T14:20:43.326 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent mirror pool info' 2024-08-31T14:20:43.327 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' mirror pool info 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mode: image 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Site Name: cluster1 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Peer Sites: 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.362 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:UUID: 39b7862d-c6c7-4e80-9773-0eeb435984a8 2024-08-31T14:20:43.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Name: cluster2 2024-08-31T14:20:43.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Mirror UUID: a8fb6c58-831b-4561-80bc-ea049221ebc5 2024-08-31T14:20:43.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Direction: rx-tx 2024-08-31T14:20:43.363 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:Client: client.mirror 2024-08-31T14:20:43.364 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent mirror pool status 2024-08-31T14:20:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent mirror pool status' 2024-08-31T14:20:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:43.365 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace '' mirror pool status --verbose 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:health: OK 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:daemon health: OK 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:image health: OK 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:images: 0 total 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:DAEMONS 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 5257: 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 5263 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.3 2024-08-31T14:20:43.402 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: true 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6024: 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6030 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.2 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.403 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:service 6093: 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: instance_id: 6099 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: client_id: mirror.1 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: hostname: smithi167 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: version: 19.3.0-4620-gdc9f1ccb 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: leader: false 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: health: OK 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.404 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:IMAGES 2024-08-31T14:20:43.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace '' ls 2024-08-31T14:20:43.442 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent rbd_mirroring omap vals 2024-08-31T14:20:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent rbd_mirroring omap vals' 2024-08-31T14:20:43.443 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace '' listomapvals rbd_mirroring 2024-08-31T14:20:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_mode 2024-08-31T14:20:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (4 bytes) : 2024-08-31T14:20:43.476 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 01 00 00 00 |....| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000004 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_peer_39b7862d-c6c7-4e80-9773-0eeb435984a8 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (132 bytes) : 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 02 01 7e 00 00 00 24 00 00 00 33 39 62 37 38 36 |..~...$...39b786| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 32 64 2d 63 36 63 37 2d 34 65 38 30 2d 39 37 37 |2d-c6c7-4e80-977| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 33 2d 30 65 65 62 34 33 35 39 38 34 61 38 08 00 |3-0eeb435984a8..| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000030 00 00 63 6c 75 73 74 65 72 32 0d 00 00 00 63 6c |..cluster2....cl| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000040 69 65 6e 74 2e 6d 69 72 72 6f 72 ff ff ff ff ff |ient.mirror.....| 2024-08-31T14:20:43.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000050 ff ff ff 02 24 00 00 00 61 38 66 62 36 63 35 38 |....$...a8fb6c58| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000060 2d 38 33 31 62 2d 34 35 36 31 2d 38 30 62 63 2d |-831b-4561-80bc-| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000070 65 61 30 34 39 32 32 31 65 62 63 35 b6 26 d3 66 |ea049221ebc5.&.f| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000080 da ae c4 0c |....| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000084 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:mirror_uuid 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (36 bytes) : 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000000 35 36 36 35 31 38 66 66 2d 37 33 32 34 2d 34 65 |566518ff-7324-4e| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000010 38 39 2d 62 62 61 38 2d 34 32 65 36 35 35 66 65 |89-bba8-42e655fe| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000020 63 30 30 32 |c002| 2024-08-31T14:20:43.481 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:00000024 2024-08-31T14:20:43.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.482 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent rbd_mirror_leader omap vals 2024-08-31T14:20:43.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent rbd_mirror_leader omap vals' 2024-08-31T14:20:43.483 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace '' listomapvals rbd_mirror_leader 2024-08-31T14:20:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_5263 2024-08-31T14:20:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6030 2024-08-31T14:20:43.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:instance_6099 2024-08-31T14:20:43.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:value (0 bytes) : 2024-08-31T14:20:43.513 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.514 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.514 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:43.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns1 images' 2024-08-31T14:20:43.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns1 images 2024-08-31T14:20:43.515 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 ls -l 2024-08-31T14:20:43.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns1' does not exist. 2024-08-31T14:20:43.549 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (2) No such file or directory 2024-08-31T14:20:43.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.551 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns1 mirror pool info 2024-08-31T14:20:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns1 mirror pool info' 2024-08-31T14:20:43.552 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 mirror pool info 2024-08-31T14:20:43.586 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns1' does not exist. 2024-08-31T14:20:43.588 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns1 mirror pool status' 2024-08-31T14:20:43.589 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns1 mirror pool status 2024-08-31T14:20:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:43.590 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 mirror pool status --verbose 2024-08-31T14:20:43.624 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns1' does not exist. 2024-08-31T14:20:43.626 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.627 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace ns1 ls 2024-08-31T14:20:43.663 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns1 rbd_mirroring omap vals 2024-08-31T14:20:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns1 rbd_mirroring omap vals' 2024-08-31T14:20:43.664 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns1 listomapvals rbd_mirroring 2024-08-31T14:20:43.698 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:error getting omap keys mirror_parent/rbd_mirroring: (2) No such file or directory 2024-08-31T14:20:43.699 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns1 rbd_mirror_leader omap vals' 2024-08-31T14:20:43.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns1 rbd_mirror_leader omap vals 2024-08-31T14:20:43.700 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns1 listomapvals rbd_mirror_leader 2024-08-31T14:20:43.734 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:error getting omap keys mirror_parent/rbd_mirror_leader: (2) No such file or directory 2024-08-31T14:20:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:20:43.736 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns2 images 2024-08-31T14:20:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns2 images' 2024-08-31T14:20:43.737 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 ls -l 2024-08-31T14:20:43.770 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns2' does not exist. 2024-08-31T14:20:43.771 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (2) No such file or directory 2024-08-31T14:20:43.772 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.773 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns2 mirror pool info 2024-08-31T14:20:43.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns2 mirror pool info' 2024-08-31T14:20:43.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 mirror pool info 2024-08-31T14:20:43.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns2' does not exist. 2024-08-31T14:20:43.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns2 mirror pool status 2024-08-31T14:20:43.811 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns2 mirror pool status' 2024-08-31T14:20:43.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:43.812 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 mirror pool status --verbose 2024-08-31T14:20:43.846 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: namespace 'ns2' does not exist. 2024-08-31T14:20:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.848 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster1 -p mirror_parent --namespace ns2 ls 2024-08-31T14:20:43.884 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns2 rbd_mirroring omap vals 2024-08-31T14:20:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns2 rbd_mirroring omap vals' 2024-08-31T14:20:43.885 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns2 listomapvals rbd_mirroring 2024-08-31T14:20:43.918 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:error getting omap keys mirror_parent/rbd_mirroring: (2) No such file or directory 2024-08-31T14:20:43.920 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster1 mirror_parent ns2 rbd_mirror_leader omap vals 2024-08-31T14:20:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster1 mirror_parent ns2 rbd_mirror_leader omap vals' 2024-08-31T14:20:43.921 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster1 -p mirror_parent --namespace ns2 listomapvals rbd_mirror_leader 2024-08-31T14:20:43.955 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:error getting omap keys mirror_parent/rbd_mirror_leader: (2) No such file or directory 2024-08-31T14:20:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:20:43.957 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:20:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for cluster in ${CLUSTER1} ${CLUSTER2} 2024-08-31T14:20:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 status 2024-08-31T14:20:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 status' 2024-08-31T14:20:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:20:43.958 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster2 -s 2024-08-31T14:25:44.029 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31T14:25:44.028+0000 7fefe01ad640 0 monclient(hunting): authenticate timed out after 300 2024-08-31T14:25:44.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:[errno 110] RADOS timed out (error connecting to the cluster) 2024-08-31T14:25:44.030 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:25:44.031 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster2 service dump 2024-08-31T14:30:44.116 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31T14:30:44.114+0000 7f6951271640 0 monclient(hunting): authenticate timed out after 300 2024-08-31T14:30:44.136 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:[errno 110] RADOS timed out (error connecting to the cluster) 2024-08-31T14:30:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:30:44.137 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ ceph --cluster cluster2 service status 2024-08-31T14:35:44.202 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:2024-08-31T14:35:44.202+0000 7f92acb7c640 0 monclient(hunting): authenticate timed out after 300 2024-08-31T14:35:44.203 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:[errno 110] RADOS timed out (error connecting to the cluster) 2024-08-31T14:35:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:35:44.204 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:35:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_pool in ${POOL} ${PARENT_POOL} 2024-08-31T14:35:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T14:35:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror images 2024-08-31T14:35:44.205 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror images' 2024-08-31T14:35:44.206 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' ls -l 2024-08-31T14:40:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T14:40:44.237 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (110) Connection timed out 2024-08-31T14:40:44.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:40:44.238 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror mirror pool info 2024-08-31T14:40:44.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:40:44.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror mirror pool info' 2024-08-31T14:40:44.239 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' mirror pool info 2024-08-31T14:44:53.335 INFO:tasks.ceph.cluster1.osd.2.smithi012.stderr:2024-08-31T14:44:53.334+0000 7f9dc3f75640 -1 reset not still connected to 0x55a0a0af4270 2024-08-31T14:45:44.272 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T14:45:44.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:45:44.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:45:44.274 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror mirror pool status 2024-08-31T14:45:44.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror mirror pool status' 2024-08-31T14:45:44.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T14:45:44.275 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace '' mirror pool status --verbose 2024-08-31T14:50:44.307 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T14:50:44.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T14:50:44.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T14:50:44.309 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster2 -p mirror --namespace '' ls 2024-08-31T14:55:44.344 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror rbd_mirroring omap vals 2024-08-31T14:55:44.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror rbd_mirroring omap vals' 2024-08-31T14:55:44.345 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace '' listomapvals rbd_mirroring 2024-08-31T15:00:44.373 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T15:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror rbd_mirror_leader omap vals 2024-08-31T15:00:44.374 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror rbd_mirror_leader omap vals' 2024-08-31T15:00:44.375 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace '' listomapvals rbd_mirror_leader 2024-08-31T15:05:44.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T15:05:44.405 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:05:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:05:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T15:05:44.406 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns1 images' 2024-08-31T15:05:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns1 images 2024-08-31T15:05:44.407 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 ls -l 2024-08-31T15:10:44.440 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:10:44.456 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (110) Connection timed out 2024-08-31T15:10:44.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:10:44.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:10:44.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns1 mirror pool info' 2024-08-31T15:10:44.457 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns1 mirror pool info 2024-08-31T15:10:44.458 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 mirror pool info 2024-08-31T15:15:44.475 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:15:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:15:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:15:44.477 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns1 mirror pool status 2024-08-31T15:15:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns1 mirror pool status' 2024-08-31T15:15:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T15:15:44.478 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns1 mirror pool status --verbose 2024-08-31T15:20:44.510 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:20:44.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:20:44.511 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:20:44.512 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster2 -p mirror --namespace ns1 ls 2024-08-31T15:25:44.546 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns1 rbd_mirroring omap vals 2024-08-31T15:25:44.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns1 rbd_mirroring omap vals' 2024-08-31T15:25:44.547 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace ns1 listomapvals rbd_mirroring 2024-08-31T15:30:44.576 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T15:30:44.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns1 rbd_mirror_leader omap vals 2024-08-31T15:30:44.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns1 rbd_mirror_leader omap vals' 2024-08-31T15:30:44.577 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace ns1 listomapvals rbd_mirror_leader 2024-08-31T15:35:44.606 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T15:35:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:35:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:35:44.607 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T15:35:44.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns2 images 2024-08-31T15:35:44.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns2 images' 2024-08-31T15:35:44.608 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 ls -l 2024-08-31T15:40:44.638 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:40:44.639 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (110) Connection timed out 2024-08-31T15:40:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:40:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:40:44.640 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns2 mirror pool info 2024-08-31T15:40:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns2 mirror pool info' 2024-08-31T15:40:44.641 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 mirror pool info 2024-08-31T15:45:44.673 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:45:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:45:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns2 mirror pool status 2024-08-31T15:45:44.675 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:45:44.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns2 mirror pool status' 2024-08-31T15:45:44.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ CEPH_ARGS= 2024-08-31T15:45:44.676 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror --namespace ns2 mirror pool status --verbose 2024-08-31T15:50:44.708 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T15:50:44.710 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T15:50:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T15:50:44.711 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:+++ rbd --cluster cluster2 -p mirror --namespace ns2 ls 2024-08-31T15:55:44.745 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns2 rbd_mirroring omap vals 2024-08-31T15:55:44.746 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns2 rbd_mirroring omap vals' 2024-08-31T15:55:44.747 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace ns2 listomapvals rbd_mirroring 2024-08-31T16:00:44.774 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T16:00:44.775 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror ns2 rbd_mirror_leader omap vals 2024-08-31T16:00:44.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror ns2 rbd_mirror_leader omap vals' 2024-08-31T16:00:44.776 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rados --cluster cluster2 -p mirror --namespace ns2 listomapvals rbd_mirror_leader 2024-08-31T16:05:44.805 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:failed to fetch mon config (--no-mon-config to skip) 2024-08-31T16:05:44.806 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T16:05:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T16:05:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_pool in ${POOL} ${PARENT_POOL} 2024-08-31T16:05:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ for image_ns in "" "${NS1}" "${NS2}" 2024-08-31T16:05:44.807 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror_parent images 2024-08-31T16:05:44.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror_parent images' 2024-08-31T16:05:44.808 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace '' ls -l 2024-08-31T16:10:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: couldn't connect to the cluster! 2024-08-31T16:10:44.838 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:rbd: listing images failed: (110) Connection timed out 2024-08-31T16:10:44.839 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout: 2024-08-31T16:10:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 2024-08-31T16:10:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stdout:cluster2 mirror_parent mirror pool info 2024-08-31T16:10:44.840 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ echo 'cluster2 mirror_parent mirror pool info' 2024-08-31T16:10:44.841 INFO:tasks.workunit.cluster1.client.mirror.smithi167.stderr:++ rbd --cluster cluster2 -p mirror_parent --namespace '' mirror pool info 2024-08-31T16:14:04.459 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2024-08-31T16:14:04.460 DEBUG:teuthology.task.console_log:Killing console logger for smithi012 2024-08-31T16:14:04.461 DEBUG:teuthology.task.console_log:Killing console logger for smithi167 2024-08-31T16:14:04.461 DEBUG:teuthology.exit:Finished running handlers